From 6d4673fd589893878d1c04a062b7021067ebbd7e Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Thu, 30 Jun 2016 16:15:08 +0200 Subject: [PATCH 001/297] Initial commit for Module to compute metrics on queries This is an initial squashed commit of the work on a new feature for query metrics proposed in #18798. --- .../support/TransportProxyClient.java | 1 + modules/rank-eval/build.gradle | 30 +++ .../index/rankeval/EvalQueryQuality.java | 45 +++++ .../index/rankeval/Evaluator.java | 28 +++ .../index/rankeval/PrecisionAtN.java | 144 ++++++++++++++ .../index/rankeval/QuerySpec.java | 114 +++++++++++ .../index/rankeval/RankEvalAction.java | 47 +++++ .../index/rankeval/RankEvalPlugin.java | 46 +++++ .../index/rankeval/RankEvalRequest.java | 69 +++++++ .../rankeval/RankEvalRequestBuilder.java | 44 +++++ .../index/rankeval/RankEvalResponse.java | 75 +++++++ .../index/rankeval/RankEvalResult.java | 73 +++++++ .../index/rankeval/RankEvalSpec.java | 109 ++++++++++ .../rankeval/RankedListQualityMetric.java | 40 ++++ .../index/rankeval/RatedQuery.java | 92 +++++++++ .../index/rankeval/RestRankEvalAction.java | 187 ++++++++++++++++++ .../rankeval/TransportRankEvalAction.java | 120 +++++++++++ .../quality/PrecisionAtRequestTests.java | 170 ++++++++++++++++ .../action/quality/RankEvalRestIT.java | 40 ++++ .../test/rank_eval/10_basic.yaml | 48 +++++ .../rest-api-spec/api/rank_eval.json | 17 ++ settings.gradle | 1 + 22 files changed, 1540 insertions(+) create mode 100644 modules/rank-eval/build.gradle create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedQuery.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/action/quality/PrecisionAtRequestTests.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java create mode 100644 modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json diff --git a/core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java b/core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java index 900876415e3..600d93e8489 100644 --- a/core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java +++ b/core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.transport.TransportService; +import java.util.Collections; import java.util.HashMap; import java.util.Map; diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle new file mode 100644 index 00000000000..cd4dd32c3d5 --- /dev/null +++ b/modules/rank-eval/build.gradle @@ -0,0 +1,30 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Rank Eval module adds APIs to evaluate ranking quality.' + classname 'org.elasticsearch.index.rankeval.RankEvalPlugin' +} + +integTest { + cluster { + setting 'script.inline', 'true' + setting 'script.stored', 'true' + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java new file mode 100644 index 00000000000..c5d48c2074a --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import java.util.Collection; + +/** Returned for each search intent and search specification combination. Summarises the document ids found that were not + * annotated and the average precision of result sets in each particular combination based on the annotations given. + * */ +public class EvalQueryQuality { + private double qualityLevel; + + private Collection unknownDocs; + + public EvalQueryQuality (double qualityLevel, Collection unknownDocs) { + this.qualityLevel = qualityLevel; + this.unknownDocs = unknownDocs; + } + + public Collection getUnknownDocs() { + return unknownDocs; + } + + public double getQualityLevel() { + return qualityLevel; + } + +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java new file mode 100644 index 00000000000..35fb4bf03cd --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java @@ -0,0 +1,28 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.search.SearchHit; + +public interface Evaluator extends NamedWriteable { + + public Object evaluate(SearchHit[] hits, RatedQuery intent); +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java new file mode 100644 index 00000000000..f6216eadede --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -0,0 +1,144 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.Map.Entry; + +import javax.naming.directory.SearchResult; + +/** + * Evaluate Precision at N, N being the number of search results to consider for precision calculation. + * + * Documents of unkonwn quality are ignored in the precision at n computation and returned by document id. + * */ +public class PrecisionAtN implements RankedListQualityMetric { + + /** Number of results to check against a given set of relevant results. */ + private int n; + + public static final String NAME = "precisionatn"; + + public PrecisionAtN(StreamInput in) throws IOException { + n = in.readInt(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(n); + } + + @Override + public String getWriteableName() { + return NAME; + } + + /** + * Initialises n with 10 + * */ + public PrecisionAtN() { + this.n = 10; + } + + /** + * @param n number of top results to check against a given set of relevant results. + * */ + public PrecisionAtN(int n) { + this.n= n; + } + + /** + * Return number of search results to check for quality. + * */ + public int getN() { + return n; + } + + /** Compute precisionAtN based on provided relevant document IDs. + * @return precision at n for above {@link SearchResult} list. + **/ + @Override + public EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent) { + Map ratedDocIds = intent.getRatedDocuments(); + + Collection relevantDocIds = new ArrayList<>(); + for (Entry entry : ratedDocIds.entrySet()) { + if (Rating.RELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) { + relevantDocIds.add(entry.getKey()); + } + } + + Collection irrelevantDocIds = new ArrayList<>(); + for (Entry entry : ratedDocIds.entrySet()) { + if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) { + irrelevantDocIds.add(entry.getKey()); + } + } + + int good = 0; + int bad = 0; + Collection unknownDocIds = new ArrayList(); + for (int i = 0; (i < n && i < hits.length); i++) { + String id = hits[i].getId(); + if (relevantDocIds.contains(id)) { + good++; + } else if (irrelevantDocIds.contains(id)) { + bad++; + } else { + unknownDocIds.add(id); + } + } + + double precision = (double) good / (good + bad); + + return new EvalQueryQuality(precision, unknownDocIds); + } + + public enum Rating { + RELEVANT, IRRELEVANT; + } + + /** + * Needed to get the enum accross serialisation boundaries. + * */ + public static class RatingMapping { + public static Integer mapFrom(Rating rating) { + if (Rating.RELEVANT.equals(rating)) { + return 0; + } + return 1; + } + + public static Rating mapTo(Integer rating) { + if (rating == 0) { + return Rating.RELEVANT; + } + return Rating.IRRELEVANT; + } + } + +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java new file mode 100644 index 00000000000..b94e0e92bd7 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java @@ -0,0 +1,114 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Defines a QA specification: All end user supplied query intents will be mapped to the search request specified in this search request + * template and executed against the targetIndex given. Any filters that should be applied in the target system can be specified as well. + * + * The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery. + * */ +public class QuerySpec implements Writeable { + + private int specId = 0; + private SearchSourceBuilder testRequest; + private List indices = new ArrayList<>(); + private List types = new ArrayList<>(); + + public QuerySpec( + int specId, SearchSourceBuilder testRequest, List indices, List types) { + this.specId = specId; + this.testRequest = testRequest; + this.indices = indices; + this.types = types; + } + + public QuerySpec(StreamInput in) throws IOException { + this.specId = in.readInt(); + testRequest = new SearchSourceBuilder(in); + int indicesSize = in.readInt(); + indices = new ArrayList(indicesSize); + for (int i = 0; i < indicesSize; i++) { + this.indices.add(in.readString()); + } + int typesSize = in.readInt(); + types = new ArrayList(typesSize); + for (int i = 0; i < typesSize; i++) { + this.types.add(in.readString()); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(specId); + testRequest.writeTo(out); + out.writeInt(indices.size()); + for (String index : indices) { + out.writeString(index); + } + out.writeInt(types.size()); + for (String type : types) { + out.writeString(type); + } + } + + public SearchSourceBuilder getTestRequest() { + return testRequest; + } + + public void setTestRequest(SearchSourceBuilder testRequest) { + this.testRequest = testRequest; + } + + public List getIndices() { + return indices; + } + + public void setIndices(List indices) { + this.indices = indices; + } + + public List getTypes() { + return types; + } + + public void setTypes(List types) { + this.types = types; + } + + /** Returns a user supplied spec id for easier referencing. */ + public int getSpecId() { + return specId; + } + + /** Sets a user supplied spec id for easier referencing. */ + public void setSpecId(int specId) { + this.specId = specId; + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java new file mode 100644 index 00000000000..0f506112d65 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; + +/** + * Action used to start precision at qa evaluations. + **/ +public class RankEvalAction extends Action { + + public static final RankEvalAction INSTANCE = new RankEvalAction(); + public static final String NAME = "indices:data/read/quality"; + + private RankEvalAction() { + super(NAME); + } + + @Override + public RankEvalRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RankEvalRequestBuilder(client, this, new RankEvalRequest()); + } + + @Override + public RankEvalResponse newResponse() { + return new RankEvalResponse(); + } + +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java new file mode 100644 index 00000000000..f0bd5c1f838 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.action.ActionModule; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.plugins.Plugin; + +public class RankEvalPlugin extends Plugin { + public static final String NAME = "rank-eval"; + + @Override + public String name() { + return NAME; + } + + @Override + public String description() { + return "The rank-eval module adds APIs to evaluate rankings."; + } + + public void onModule(ActionModule actionModule) { + actionModule.registerAction(RankEvalAction.INSTANCE, TransportRankEvalAction.class); + } + + public void onModule(NetworkModule networkModule) { + networkModule.registerRestHandler(RestRankEvalAction.class); + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java new file mode 100644 index 00000000000..3e8f893faaa --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + + +/** + * Instances of this class represent a complete precision at request. They encode a precision task including search intents and search + * specifications to be executed subsequently. + * */ +public class RankEvalRequest extends ActionRequest { + + /** The request data to use for evaluation. */ + private RankEvalSpec task; + + @Override + public ActionRequestValidationException validate() { + return null; // TODO + } + + /** Returns the specification of this qa run including intents to execute, specifications detailing intent translation and metrics + * to compute. */ + public RankEvalSpec getRankEvalSpec() { + return task; + } + + /** Returns the specification of this qa run including intents to execute, specifications detailing intent translation and metrics + * to compute. */ + public void setRankEvalSpec(RankEvalSpec task) { + this.task = task; + } + + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + task = new RankEvalSpec(in); + + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + task.writeTo(out); + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java new file mode 100644 index 00000000000..063bec9d8f7 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; + +public class RankEvalRequestBuilder extends ActionRequestBuilder { + + public RankEvalRequestBuilder(ElasticsearchClient client, Action action, + RankEvalRequest request) { + super(client, action, request); + } + + public RankEvalRequest request() { + return request; + } + + public void setRankEvalSpec(RankEvalSpec spec) { + this.request.setRankEvalSpec(spec); + } + + public RankEvalSpec getRankEvalSpec() { + return this.request.getRankEvalSpec(); + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java new file mode 100644 index 00000000000..f4ab789d429 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -0,0 +1,75 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + +/** + * For each qa specification identified by its id this response returns the respective + * averaged precisionAnN value. + * + * In addition for each query the document ids that haven't been found annotated is returned as well. + * + * Documents of unknown quality - i.e. those that haven't been supplied in the set of annotated documents but have been returned + * by the search are not taken into consideration when computing precision at n - they are ignored. + * + **/ +public class RankEvalResponse extends ActionResponse { + + private Collection qualityResults = new ArrayList<>(); + + public RankEvalResponse() { + + } + + public RankEvalResponse(StreamInput in) throws IOException { + int size = in.readInt(); + qualityResults = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + qualityResults.add(new RankEvalResult(in)); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeInt(qualityResults.size()); + for (RankEvalResult result : qualityResults) { + result.writeTo(out); + } + } + + public void addRankEvalResult(int specId, double quality, Map> unknownDocs) { + RankEvalResult result = new RankEvalResult(specId, quality, unknownDocs); + this.qualityResults.add(result); + } + + public Collection getRankEvalResults() { + return qualityResults; + } + +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java new file mode 100644 index 00000000000..b5bae8b6d4c --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; + +/** + * For each precision at n computation the id of the search request specification used to generate search requests is returned + * for reference. In addition the averaged precision and the ids of all documents returned but not found annotated is returned. + * */ +public class RankEvalResult implements Writeable { + /**ID of specification this result was generated for.*/ + private int specId; + /**Average precision observed when issueing query intents with this spec.*/ + private double qualityLevel; + /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ + private Map> unknownDocs; + + @SuppressWarnings("unchecked") + public RankEvalResult(StreamInput in) throws IOException { + this.specId = in.readInt(); + this.qualityLevel = in.readDouble(); + this.unknownDocs = (Map>) in.readGenericValue(); + } + + public RankEvalResult(int specId, double quality, Map> unknownDocs) { + this.specId = specId; + this.qualityLevel = quality; + this.unknownDocs = unknownDocs; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(specId); + out.writeDouble(qualityLevel); + out.writeGenericValue(getUnknownDocs()); + } + + public int getSpecId() { + return specId; + } + + public double getQualityLevel() { + return qualityLevel; + } + + public Map> getUnknownDocs() { + return unknownDocs; + } +} \ No newline at end of file diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java new file mode 100644 index 00000000000..f625401be53 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -0,0 +1,109 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + +/** + * This class defines a qa task including query intent and query spec. + * + * Each QA run is based on a set of queries to send to the index and multiple QA specifications that define how to translate the query + * intents into elastic search queries. In addition it contains the quality metrics to compute. + * */ + +public class RankEvalSpec implements Writeable { + /** Collection of query intents to check against including expected document ids.*/ + private Collection intents = new ArrayList<>(); + /** Collection of query specifications, that is e.g. search request templates to use for query translation. */ + private Collection specifications = new ArrayList<>(); + /** Definition of n in precision at n */ + private RankedListQualityMetric eval; + + + public RankEvalSpec(Collection intents, Collection specs, RankedListQualityMetric metric) { + this.intents = intents; + this.specifications = specs; + this.eval = metric; + } + + public RankEvalSpec(StreamInput in) throws IOException { + int intentSize = in.readInt(); + intents = new ArrayList<>(intentSize); + for (int i = 0; i < intentSize; i++) { + intents.add(new RatedQuery(in)); + } + int specSize = in.readInt(); + specifications = new ArrayList<>(specSize); + for (int i = 0; i < specSize; i++) { + specifications.add(new QuerySpec(in)); + } + eval = in.readNamedWriteable(RankedListQualityMetric.class); // TODO add to registry + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(intents.size()); + for (RatedQuery query : intents) { + query.writeTo(out); + } + out.writeInt(specifications.size()); + for (QuerySpec spec : specifications) { + spec.writeTo(out); + } + out.writeNamedWriteable(eval); + } + + /** Returns the precision at n configuration (containing level of n to consider).*/ + public RankedListQualityMetric getEvaluator() { + return eval; + } + + /** Sets the precision at n configuration (containing level of n to consider).*/ + public void setEvaluator(RankedListQualityMetric config) { + this.eval = config; + } + + /** Returns a list of search intents to evaluate. */ + public Collection getIntents() { + return intents; + } + + /** Set a list of search intents to evaluate. */ + public void setIntents(Collection intents) { + this.intents = intents; + } + + /** Returns a list of intent to query translation specifications to evaluate. */ + public Collection getSpecifications() { + return specifications; + } + + /** Set the list of intent to query translation specifications to evaluate. */ + public void setSpecifications(Collection specifications) { + this.specifications = specifications; + } + +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java new file mode 100644 index 00000000000..1a75247e48d --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.search.SearchHit; + +/** + * Classes implementing this interface provide a means to compute the quality of a result list + * returned by some search. + * + * RelevancyLevel specifies the type of object determining the relevancy level of some known docid. + * */ +public interface RankedListQualityMetric extends Evaluator { + + /** + * Returns a single metric representing the ranking quality of a set of returned documents + * wrt. to a set of document Ids labeled as relevant for this search. + * + * @param hits the result hits as returned by some search + * @return some metric representing the quality of the result hit list wrt. to relevant doc ids. + * */ + public EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent); +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedQuery.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedQuery.java new file mode 100644 index 00000000000..5c4c7ea1e53 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedQuery.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Objects of this class represent one type of user query to qa. Each query comprises a user supplied id for easer referencing, + * a set of parameters as supplied by the end user to the search application as well as a set of rated documents (ratings e.g. + * supplied by manual result tagging or some form of automated click log based process). + * */ +public class RatedQuery implements Writeable { + + private final int intentId; + private final Map intentParameters; + private final Map ratedDocuments; + + public RatedQuery( + int intentId, Map intentParameters, Map ratedDocuments) { + this.intentId = intentId; + this.intentParameters = intentParameters; + this.ratedDocuments = ratedDocuments; + } + + public RatedQuery(StreamInput in) throws IOException { + this.intentId = in.readInt(); + this.intentParameters = in.readMap(); + + int ratedDocsSize = in.readInt(); + this.ratedDocuments = new HashMap<>(ratedDocsSize); + for (int i = 0; i < ratedDocsSize; i++) { + this.ratedDocuments.put(in.readString(), in.readInt()); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(intentId); + out.writeMap(intentParameters); + out.writeInt(ratedDocuments.size()); + for(Entry entry : ratedDocuments.entrySet()) { + out.writeString(entry.getKey()); + out.writeInt(entry.getValue()); + } + } + + /** For easier referencing users are allowed to supply unique ids with each search intent they want to check for + * performance quality wise.*/ + public int getIntentId() { + return intentId; + } + + + /** + * Returns a mapping from query parameter name to real parameter - ideally as parsed from real user logs. + * */ + public Map getIntentParameters() { + return intentParameters; + } + + /** + * Returns a set of documents and their ratings as supplied by the users. + * */ + public Map getRatedDocuments() { + return ratedDocuments; + } + +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java new file mode 100644 index 00000000000..b8093692130 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -0,0 +1,187 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.support.RestActions; +import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.suggest.Suggesters; + +import java.io.IOException; + +import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.rest.RestRequest.Method.POST; + +/** + * Accepted input format: + * + * General Format: + * + * + { "requests": [{ + "id": "human_readable_id", + "request": { ... request to check ... }, + "ratings": { ... mapping from doc id to rating value ... } + }], + "metric": { + "... metric_name... ": { + "... metric_parameter_key ...": ...metric_parameter_value... + }}} + * + * Example: + * + * + {"requests": [{ + "id": "amsterdam_query", + "request": { + "query": { + "bool": { + "must": [ + {"match": {"beverage": "coffee"}}, + {"term": {"browser": {"value": "safari"}}}, + {"term": {"time_of_day": {"value": "morning","boost": 2}}}, + {"term": {"ip_location": {"value": "ams","boost": 10}}}]} + }, + "size": 10 + } + }, + "ratings": { + "1": 1, + "2": 0, + "3": 1, + "4": 1 + } + }, { + "id": "berlin_query", + "request": { + "query": { + "bool": { + "must": [ + {"match": {"beverage": "club mate"}}, + {"term": {"browser": {"value": "chromium"}}}, + {"term": {"time_of_day": {"value": "evening","boost": 2}}}, + {"term": {"ip_location": {"value": "ber","boost": 10}}}]} + }, + "size": 10 + } + }, + "ratings": { + "1": 0, + "5": 1, + "6": 1 + } + }], + "metric": { + "precisionAtN": { + "size": 10}} + } + + * + * Output format: + * + * General format: + * + * + { + "took": 59, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "failed": 0 + }, + "quality_level": ... quality level ..., + "unknown_docs": [{"user_request_id": [... list of unknown docs ...]}] +} + + * + * Example: + * + * + * + { + "took": 59, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "failed": 0 + }, + "rank_eval": [{ + "spec_id": "huge_weight_on_city", + "quality_level": 0.4, + "unknown_docs": [{ + "amsterdam_query": [5, 10, 23] + }, { + "berlin_query": [42] + }] + }] + } + + + * */ +public class RestRankEvalAction extends BaseRestHandler { + + @Inject + public RestRankEvalAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry, + AggregatorParsers aggParsers, Suggesters suggesters) { + super(settings, client); + controller.registerHandler(GET, "/_rank_eval", this); + controller.registerHandler(POST, "/_rank_eval", this); + controller.registerHandler(GET, "/{index}/_rank_eval", this); + controller.registerHandler(POST, "/{index}/_rank_eval", this); + controller.registerHandler(GET, "/{index}/{type}/_rank_eval", this); + controller.registerHandler(POST, "/{index}/{type}/_rank_eval", this); + } + + @Override + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException { + RankEvalRequest rankEvalRequest = new RankEvalRequest(); + //parseRankEvalRequest(rankEvalRequest, request, parseFieldMatcher); + //client.rankEval(rankEvalRequest, new RestStatusToXContentListener<>(channel)); + } + + public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, ParseFieldMatcher parseFieldMatcher) + throws IOException { + + String[] indices = Strings.splitStringByCommaToArray(request.param("index")); + BytesReference restContent = null; + if (restContent == null) { + if (RestActions.hasBodyContent(request)) { + restContent = RestActions.getRestContent(request); + } + } + if (restContent != null) { + } + + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java new file mode 100644 index 00000000000..f75bf3a0ebd --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -0,0 +1,120 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.AutoCreateIndex; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.action.SearchTransportService; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.controller.SearchPhaseController; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * Instances of this class execute a collection of search intents (read: user supplied query parameters) against a set of + * possible search requests (read: search specifications, expressed as query/search request templates) and compares the result + * against a set of annotated documents per search intent. + * + * If any documents are returned that haven't been annotated the document id of those is returned per search intent. + * + * The resulting search quality is computed in terms of precision at n and returned for each search specification for the full + * set of search intents as averaged precision at n. + * */ +public class TransportRankEvalAction extends HandledTransportAction { + private SearchPhaseController searchPhaseController; + private TransportService transportService; + private SearchTransportService searchTransportService; + private ClusterService clusterService; + private ActionFilters actionFilters; + + @Inject + public TransportRankEvalAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, ScriptService scriptService, + AutoCreateIndex autoCreateIndex, Client client, TransportService transportService, SearchPhaseController searchPhaseController, + SearchTransportService searchTransportService, NamedWriteableRegistry namedWriteableRegistry) { + super(settings, RankEvalAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, + RankEvalRequest::new); + this.searchPhaseController = searchPhaseController; + this.transportService = transportService; + this.searchTransportService = searchTransportService; + this.clusterService = clusterService; + this.actionFilters = actionFilters; + + namedWriteableRegistry.register(RankedListQualityMetric.class, PrecisionAtN.NAME, PrecisionAtN::new); + } + + @Override + protected void doExecute(RankEvalRequest request, ActionListener listener) { + RankEvalResponse response = new RankEvalResponse(); + RankEvalSpec qualityTask = request.getRankEvalSpec(); + RankedListQualityMetric metric = qualityTask.getEvaluator(); + + for (QuerySpec spec : qualityTask.getSpecifications()) { + double qualitySum = 0; + + SearchSourceBuilder specRequest = spec.getTestRequest(); + String[] indices = new String[spec.getIndices().size()]; + spec.getIndices().toArray(indices); + SearchRequest templatedRequest = new SearchRequest(indices, specRequest); + + + Map> unknownDocs = new HashMap>(); + Collection intents = qualityTask.getIntents(); + for (RatedQuery intent : intents) { + + TransportSearchAction transportSearchAction = new TransportSearchAction( + settings, + threadPool, + searchPhaseController, + transportService, + searchTransportService, + clusterService, + actionFilters, + indexNameExpressionResolver); + ActionFuture searchResponse = transportSearchAction.execute(templatedRequest); + SearchHits hits = searchResponse.actionGet().getHits(); + + EvalQueryQuality intentQuality = metric.evaluate(hits.getHits(), intent); + qualitySum += intentQuality.getQualityLevel(); + unknownDocs.put(intent.getIntentId(), intentQuality.getUnknownDocs()); + } + response.addRankEvalResult(spec.getSpecId(), qualitySum / intents.size(), unknownDocs); + } + listener.onResponse(response); + } +} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/PrecisionAtRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/PrecisionAtRequestTests.java new file mode 100644 index 00000000000..c6a9fa658d0 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/PrecisionAtRequestTests.java @@ -0,0 +1,170 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.quality; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.rankeval.PrecisionAtN; +import org.elasticsearch.index.rankeval.RankEvalPlugin; +import org.elasticsearch.index.rankeval.RatedQuery; +import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.test.ESIntegTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, transportClientRatio = 0.0) +// NORELEASE need to fix transport client use case +public class PrecisionAtRequestTests extends ESIntegTestCase { + @Override + protected Collection> transportClientPlugins() { + return pluginList(RankEvalPlugin.class); + } + + @Override + protected Collection> nodePlugins() { + return pluginList(RankEvalPlugin.class); + } + + @Before + public void setup() { + createIndex("test"); + ensureGreen(); + + client().prepareIndex("test", "testtype").setId("1") + .setSource("text", "berlin").get(); + client().prepareIndex("test", "testtype").setId("2") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("3") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("4") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("5") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("6") + .setSource("text", "amsterdam").get(); + refresh(); + } + + + public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { + // TODO turn into unit test - no need to execute the query here to fill hits object + MatchQueryBuilder query = new MatchQueryBuilder("text", "berlin"); + + SearchResponse response = client().prepareSearch().setQuery(query) + .execute().actionGet(); + + Map relevant = new HashMap<>(); + relevant.put("1", Rating.RELEVANT.ordinal()); + RatedQuery intent = new RatedQuery(0, new HashMap<>(), relevant); + SearchHit[] hits = response.getHits().getHits(); + + assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, intent).getQualityLevel(), 0.00001); + } + + public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException { + // TODO turn into unit test - no need to actually execute the query here to fill the hits object + MatchQueryBuilder query = new MatchQueryBuilder("text", "amsterdam"); + + SearchResponse response = client().prepareSearch().setQuery(query) + .execute().actionGet(); + + Map relevant = new HashMap<>(); + relevant.put("2", Rating.RELEVANT.ordinal()); + relevant.put("3", Rating.RELEVANT.ordinal()); + relevant.put("4", Rating.RELEVANT.ordinal()); + relevant.put("5", Rating.RELEVANT.ordinal()); + relevant.put("6", Rating.IRRELEVANT.ordinal()); + RatedQuery intent = new RatedQuery(0, new HashMap<>(), relevant); + SearchHit[] hits = response.getHits().getHits(); + + assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, intent).getQualityLevel(), 0.00001); + } + + public void testPrecisionJSON() { + + } + +/* public void testPrecisionAction() { + // TODO turn into REST test? + + Collection intents = new ArrayList(); + RatedQuery intentAmsterdam = new RatedQuery( + 0, + createParams("var", "amsterdam"), + createRelevant("2", "3", "4", "5")); + intents.add(intentAmsterdam); + + RatedQuery intentBerlin = new RatedQuery( + 1, + createParams("var", "berlin"), + createRelevant("1")); + intents.add(intentBerlin); + + Collection specs = new ArrayList(); + ArrayList indices = new ArrayList<>(); + indices.add("test"); + ArrayList types = new ArrayList<>(); + types.add("testtype"); + + SearchSourceBuilder source = new SearchSourceBuilder(); + QuerySpec spec = new QuerySpec(0, source, indices, types); + specs.add(spec); + + RankEvalSpec task = new RankEvalSpec(intents, specs, new PrecisionAtN(10)); + + RankEvalRequestBuilder builder = new RankEvalRequestBuilder( + client(), + RankEvalAction.INSTANCE, + new RankEvalRequest()); + builder.setRankEvalSpec(task); + + RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + RankEvalResult result = response.getRankEvalResults().iterator().next(); + for (Entry> entry : result.getUnknownDocs().entrySet()) { + if (entry.getKey() == 0) { + assertEquals(1, entry.getValue().size()); + } else { + assertEquals(0, entry.getValue().size()); + } + } + }*/ + + private Map createRelevant(String... docs) { + Map relevant = new HashMap<>(); + for (String doc : docs) { + relevant.put(doc, Rating.RELEVANT.ordinal()); + } + return relevant; + } + + private Map createParams(String key, String value) { + Map parameters = new HashMap<>(); + parameters.put(key, value); + return parameters; + } + + } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java new file mode 100644 index 00000000000..b68385017f6 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.quality; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; + +import java.io.IOException; + +public class RankEvalRestIT extends ESRestTestCase { + public RankEvalRestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return ESRestTestCase.createParameters(0, 1); + } +} diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml new file mode 100644 index 00000000000..201efde33aa --- /dev/null +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -0,0 +1,48 @@ +--- +"Response format": + + - do: + index: + index: foo + type: bar + id: 1 + body: { "text": "berlin" } + + - do: + index: + index: foo + type: bar + id: 2 + body: { "text": "amsterdam" } + + - do: + index: + index: foo + type: bar + id: 3 + body: { "text": "amsterdam" } + + - do: + indices.refresh: {} + + - do: + rank_eval: + body: + requests: [ + { + id: "amsterdam_query", + request: { query: {match : {text : "amsterdam" }}}, + ratings: { "1": 0, "2": 1, "3": 1 } + }, { + id: "berlin_query", + request: { query: { match : { text : "berlin" } }, size : 10 }, + ratings: {"1": 1} + } + ] + metric: { precisionAtN: { size: 10}} + + - match: {quality_level: 1} + - gte: { took: 0 } + - is_false: task + - is_false: deleted + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json new file mode 100644 index 00000000000..681eb9f6081 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json @@ -0,0 +1,17 @@ +{ + "rank_eval": { + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-rank-eval.html", + "methods": ["POST"], + "url": { + "path": "/_rank_eval", + "paths": ["/_rank_eval"], + "parts": {}, + "params": {} + }, + "body": { + "description": "The search definition using the Query DSL and the prototype for the eval request.", + "required": true + } + } +} + diff --git a/settings.gradle b/settings.gradle index 6588b605a9d..982fc004a18 100644 --- a/settings.gradle +++ b/settings.gradle @@ -25,6 +25,7 @@ List projects = [ 'modules:lang-mustache', 'modules:lang-painless', 'modules:reindex', + 'modules:rank-eval', 'modules:percolator', 'plugins:analysis-icu', 'plugins:analysis-kuromoji', From be19d13d14fae373d1625e065466d334660dd080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 30 Jun 2016 15:05:51 +0200 Subject: [PATCH 002/297] Adapting to changes in Plugin api --- .../index/rankeval/RankEvalPlugin.java | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index f0bd5c1f838..7aed80fc481 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -19,28 +19,24 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.action.ActionModule; -import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestHandler; -public class RankEvalPlugin extends Plugin { - public static final String NAME = "rank-eval"; +import java.util.Arrays; +import java.util.List; + +public class RankEvalPlugin extends Plugin implements ActionPlugin { @Override - public String name() { - return NAME; + public List, ? extends ActionResponse>> getActions() { + return Arrays.asList(new ActionHandler<>(RankEvalAction.INSTANCE, TransportRankEvalAction.class)); } @Override - public String description() { - return "The rank-eval module adds APIs to evaluate rankings."; - } - - public void onModule(ActionModule actionModule) { - actionModule.registerAction(RankEvalAction.INSTANCE, TransportRankEvalAction.class); - } - - public void onModule(NetworkModule networkModule) { - networkModule.registerRestHandler(RestRankEvalAction.class); + public List> getRestHandlers() { + return Arrays.asList(RestRankEvalAction.class); } } From 42662fd6951d8c5ad3bb3c71ee3191e0fcf9dd8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 1 Jul 2016 15:03:17 +0200 Subject: [PATCH 003/297] Adapt to changes in master --- .../index/rankeval/RestRankEvalAction.java | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index b8093692130..196fb9c5a9a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -19,14 +19,12 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.client.Client; +import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; @@ -43,10 +41,10 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; /** * Accepted input format: - * + * * General Format: - * - * + * + * { "requests": [{ "id": "human_readable_id", "request": { ... request to check ... }, @@ -56,17 +54,17 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; "... metric_name... ": { "... metric_parameter_key ...": ...metric_parameter_value... }}} - * - * Example: - * - * + * + * Example: + * + * {"requests": [{ "id": "amsterdam_query", "request": { "query": { "bool": { "must": [ - {"match": {"beverage": "coffee"}}, + {"match": {"beverage": "coffee"}}, {"term": {"browser": {"value": "safari"}}}, {"term": {"time_of_day": {"value": "morning","boost": 2}}}, {"term": {"ip_location": {"value": "ams","boost": 10}}}]} @@ -86,7 +84,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; "query": { "bool": { "must": [ - {"match": {"beverage": "club mate"}}, + {"match": {"beverage": "club mate"}}, {"term": {"browser": {"value": "chromium"}}}, {"term": {"time_of_day": {"value": "evening","boost": 2}}}, {"term": {"ip_location": {"value": "ber","boost": 10}}}]} @@ -103,14 +101,14 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; "metric": { "precisionAtN": { "size": 10}} - } - - * + } + + * * Output format: - * + * * General format: - * - * + * + * { "took": 59, "timed_out": false, @@ -121,13 +119,13 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; }, "quality_level": ... quality level ..., "unknown_docs": [{"user_request_id": [... list of unknown docs ...]}] -} +} - * + * * Example: - * - * - * + * + * + * { "took": 59, "timed_out": false, @@ -146,15 +144,15 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; }] }] } - - + + * */ public class RestRankEvalAction extends BaseRestHandler { @Inject - public RestRankEvalAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry, + public RestRankEvalAction(Settings settings, RestController controller, IndicesQueriesRegistry queryRegistry, AggregatorParsers aggParsers, Suggesters suggesters) { - super(settings, client); + super(settings); controller.registerHandler(GET, "/_rank_eval", this); controller.registerHandler(POST, "/_rank_eval", this); controller.registerHandler(GET, "/{index}/_rank_eval", this); @@ -164,7 +162,7 @@ public class RestRankEvalAction extends BaseRestHandler { } @Override - public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException { + public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws IOException { RankEvalRequest rankEvalRequest = new RankEvalRequest(); //parseRankEvalRequest(rankEvalRequest, request, parseFieldMatcher); //client.rankEval(rankEvalRequest, new RestStatusToXContentListener<>(channel)); From b38a12ad15028d4138745e4430e630d9c4623f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 5 Jul 2016 11:38:19 +0200 Subject: [PATCH 004/297] Adapt to new checkstyle constraints --- .../java/org/elasticsearch/index/rankeval/Evaluator.java | 4 ++-- .../index/rankeval/RankedListQualityMetric.java | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java index 35fb4bf03cd..dba7403c652 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java @@ -23,6 +23,6 @@ import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.search.SearchHit; public interface Evaluator extends NamedWriteable { - - public Object evaluate(SearchHit[] hits, RatedQuery intent); + + Object evaluate(SearchHit[] hits, RatedQuery intent); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index 1a75247e48d..41168b10329 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -24,7 +24,7 @@ import org.elasticsearch.search.SearchHit; /** * Classes implementing this interface provide a means to compute the quality of a result list * returned by some search. - * + * * RelevancyLevel specifies the type of object determining the relevancy level of some known docid. * */ public interface RankedListQualityMetric extends Evaluator { @@ -36,5 +36,6 @@ public interface RankedListQualityMetric extends Evaluator { * @param hits the result hits as returned by some search * @return some metric representing the quality of the result hit list wrt. to relevant doc ids. * */ - public EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent); + @Override + EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent); } From b730494bfc904bcb1dadc9d3c1d58992333556d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 4 Jul 2016 12:57:18 +0200 Subject: [PATCH 005/297] Adding rest layer parsing and response rendering Adding parsers for the rest request and the various components within, also extending the existing rest test and adding rendering of the response. --- .../index/rankeval/EvalQueryQuality.java | 10 +- .../index/rankeval/Evaluator.java | 28 --- .../index/rankeval/PrecisionAtN.java | 66 ++++--- .../index/rankeval/QuerySpec.java | 96 +++++++++- .../index/rankeval/RankEvalContext.java | 65 +++++++ .../index/rankeval/RankEvalRequest.java | 16 +- .../index/rankeval/RankEvalResponse.java | 63 ++++--- .../index/rankeval/RankEvalResult.java | 30 ++-- .../index/rankeval/RankEvalSpec.java | 47 +++-- .../rankeval/RankedListQualityMetric.java | 35 +++- .../index/rankeval/RatedDocument.java | 86 +++++++++ .../index/rankeval/RatedQuery.java | 92 ---------- .../index/rankeval/RestRankEvalAction.java | 88 ++++++--- .../rankeval/TransportRankEvalAction.java | 56 +++--- .../quality/PrecisionAtRequestTests.java | 170 ------------------ .../action/quality/RankEvalRequestTests.java | 122 +++++++++++++ .../index/rankeval/PrecisionAtNTests.java | 69 +++++++ .../index/rankeval/QuerySpecTests.java | 98 ++++++++++ .../test/rank_eval/10_basic.yaml | 49 ++--- 19 files changed, 810 insertions(+), 476 deletions(-) delete mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java delete mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedQuery.java delete mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/action/quality/PrecisionAtRequestTests.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index c5d48c2074a..54edd722126 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -21,25 +21,25 @@ package org.elasticsearch.index.rankeval; import java.util.Collection; -/** Returned for each search intent and search specification combination. Summarises the document ids found that were not - * annotated and the average precision of result sets in each particular combination based on the annotations given. +/** Returned for each search specification. Summarizes the measured quality metric for this search request + * and adds the document ids found that were in the search result but not annotated in the original request. * */ public class EvalQueryQuality { private double qualityLevel; - + private Collection unknownDocs; public EvalQueryQuality (double qualityLevel, Collection unknownDocs) { this.qualityLevel = qualityLevel; this.unknownDocs = unknownDocs; } - + public Collection getUnknownDocs() { return unknownDocs; } public double getQualityLevel() { - return qualityLevel; + return qualityLevel; } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java deleted file mode 100644 index dba7403c652..00000000000 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.rankeval; - -import org.elasticsearch.common.io.stream.NamedWriteable; -import org.elasticsearch.search.SearchHit; - -public interface Evaluator extends NamedWriteable { - - Object evaluate(SearchHit[] hits, RatedQuery intent); -} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index f6216eadede..1d67c45dbb8 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -19,28 +19,31 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Map; -import java.util.Map.Entry; +import java.util.List; import javax.naming.directory.SearchResult; /** * Evaluate Precision at N, N being the number of search results to consider for precision calculation. - * + * * Documents of unkonwn quality are ignored in the precision at n computation and returned by document id. * */ -public class PrecisionAtN implements RankedListQualityMetric { - +public class PrecisionAtN extends RankedListQualityMetric { + /** Number of results to check against a given set of relevant results. */ private int n; - + public static final String NAME = "precisionatn"; public PrecisionAtN(StreamInput in) throws IOException { @@ -63,7 +66,7 @@ public class PrecisionAtN implements RankedListQualityMetric { public PrecisionAtN() { this.n = 10; } - + /** * @param n number of top results to check against a given set of relevant results. * */ @@ -78,24 +81,31 @@ public class PrecisionAtN implements RankedListQualityMetric { return n; } + private static final ParseField SIZE_FIELD = new ParseField("size"); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "precision_at", a -> new PrecisionAtN((Integer) a[0])); + + static { + PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD); + } + + public static PrecisionAtN fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { + return PARSER.apply(parser, matcher); + } + /** Compute precisionAtN based on provided relevant document IDs. * @return precision at n for above {@link SearchResult} list. **/ @Override - public EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent) { - Map ratedDocIds = intent.getRatedDocuments(); - - Collection relevantDocIds = new ArrayList<>(); - for (Entry entry : ratedDocIds.entrySet()) { - if (Rating.RELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) { - relevantDocIds.add(entry.getKey()); - } - } - - Collection irrelevantDocIds = new ArrayList<>(); - for (Entry entry : ratedDocIds.entrySet()) { - if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) { - irrelevantDocIds.add(entry.getKey()); + public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { + + Collection relevantDocIds = new ArrayList<>(); + Collection irrelevantDocIds = new ArrayList<>(); + for (RatedDocument doc : ratedDocs) { + if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + relevantDocIds.add(doc.getDocID()); + } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + irrelevantDocIds.add(doc.getDocID()); } } @@ -117,24 +127,24 @@ public class PrecisionAtN implements RankedListQualityMetric { return new EvalQueryQuality(precision, unknownDocIds); } - + public enum Rating { - RELEVANT, IRRELEVANT; + IRRELEVANT, RELEVANT; } - + /** * Needed to get the enum accross serialisation boundaries. * */ public static class RatingMapping { public static Integer mapFrom(Rating rating) { if (Rating.RELEVANT.equals(rating)) { - return 0; + return 1; } - return 1; + return 0; } - + public static Rating mapTo(Integer rating) { - if (rating == 0) { + if (rating == 1) { return Rating.RELEVANT; } return Rating.IRRELEVANT; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java index b94e0e92bd7..2e82fd98939 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java @@ -19,9 +19,13 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; @@ -32,25 +36,33 @@ import java.util.List; * Defines a QA specification: All end user supplied query intents will be mapped to the search request specified in this search request * template and executed against the targetIndex given. Any filters that should be applied in the target system can be specified as well. * - * The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery. + * The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery. * */ public class QuerySpec implements Writeable { - private int specId = 0; + private String specId; private SearchSourceBuilder testRequest; private List indices = new ArrayList<>(); private List types = new ArrayList<>(); - - public QuerySpec( - int specId, SearchSourceBuilder testRequest, List indices, List types) { + /** Collection of rated queries for this query QA specification.*/ + private List ratedDocs = new ArrayList<>(); + + public QuerySpec() { + // ctor that doesn't require all args to be present immediatly is easier to use with ObjectParser + // TODO decide if we can require only id as mandatory, set default values for the rest? + } + + public QuerySpec(String specId, SearchSourceBuilder testRequest, List indices, List types, + List ratedDocs) { this.specId = specId; this.testRequest = testRequest; this.indices = indices; this.types = types; + this.ratedDocs = ratedDocs; } public QuerySpec(StreamInput in) throws IOException { - this.specId = in.readInt(); + this.specId = in.readString(); testRequest = new SearchSourceBuilder(in); int indicesSize = in.readInt(); indices = new ArrayList(indicesSize); @@ -62,11 +74,16 @@ public class QuerySpec implements Writeable { for (int i = 0; i < typesSize; i++) { this.types.add(in.readString()); } + int intentSize = in.readInt(); + ratedDocs = new ArrayList<>(intentSize); + for (int i = 0; i < intentSize; i++) { + ratedDocs.add(new RatedDocument(in)); + } } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeInt(specId); + out.writeString(specId); testRequest.writeTo(out); out.writeInt(indices.size()); for (String index : indices) { @@ -76,6 +93,10 @@ public class QuerySpec implements Writeable { for (String type : types) { out.writeString(type); } + out.writeInt(ratedDocs.size()); + for (RatedDocument ratedDoc : ratedDocs) { + ratedDoc.writeTo(out); + } } public SearchSourceBuilder getTestRequest() { @@ -103,12 +124,69 @@ public class QuerySpec implements Writeable { } /** Returns a user supplied spec id for easier referencing. */ - public int getSpecId() { + public String getSpecId() { return specId; } /** Sets a user supplied spec id for easier referencing. */ - public void setSpecId(int specId) { + public void setSpecId(String specId) { this.specId = specId; } + + /** Returns a list of rated documents to evaluate. */ + public List getRatedDocs() { + return ratedDocs; + } + + /** Set a list of rated documents for this query. */ + public void setRatedDocs(List ratedDocs) { + this.ratedDocs = ratedDocs; + } + + private static final ParseField ID_FIELD = new ParseField("id"); + private static final ParseField REQUEST_FIELD = new ParseField("request"); + private static final ParseField RATINGS_FIELD = new ParseField("ratings"); + private static final ObjectParser PARSER = new ObjectParser<>("requests", QuerySpec::new); + + static { + PARSER.declareString(QuerySpec::setSpecId, ID_FIELD); + PARSER.declareObject(QuerySpec::setTestRequest, (p, c) -> { + try { + return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters()); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing request", ex); + } + } , REQUEST_FIELD); + PARSER.declareObjectArray(QuerySpec::setRatedDocs, (p, c) -> { + try { + return RatedDocument.fromXContent(p); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex); + } + } , RATINGS_FIELD); + } + + /** + * Parses {@link QuerySpec} from rest representation: + * + * Example: + * { + * "id": "coffee_query", + * "request": { + * "query": { + * "bool": { + * "must": [ + * {"match": {"beverage": "coffee"}}, + * {"term": {"browser": {"value": "safari"}}}, + * {"term": {"time_of_day": {"value": "morning","boost": 2}}}, + * {"term": {"ip_location": {"value": "ams","boost": 10}}}]} + * }, + * "size": 10 + * }, + * "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ] + * } + */ + public static QuerySpec fromXContent(XContentParser parser, RankEvalContext context) throws IOException { + return PARSER.parse(parser, context); + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java new file mode 100644 index 00000000000..780585d978d --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.suggest.Suggesters; + +public class RankEvalContext implements ParseFieldMatcherSupplier { + + private final ParseFieldMatcher parseFieldMatcher; + private final AggregatorParsers aggs; + private final Suggesters suggesters; + private final QueryParseContext parseContext; + + public RankEvalContext(ParseFieldMatcher parseFieldMatcher, QueryParseContext parseContext, AggregatorParsers aggs, + Suggesters suggesters) { + this.parseFieldMatcher = parseFieldMatcher; + this.aggs = aggs; + this.suggesters = suggesters; + this.parseContext = parseContext; + } + + public Suggesters getSuggesters() { + return this.suggesters; + } + + public AggregatorParsers getAggs() { + return this.aggs; + } + + @Override + public ParseFieldMatcher getParseFieldMatcher() { + return this.parseFieldMatcher; + } + + public XContentParser parser() { + return this.parseContext.parser(); + } + + public QueryParseContext getParseContext() { + return this.parseContext; + } + +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java index 3e8f893faaa..8dbbef3b497 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java @@ -35,20 +35,24 @@ public class RankEvalRequest extends ActionRequest { /** The request data to use for evaluation. */ private RankEvalSpec task; - + @Override public ActionRequestValidationException validate() { - return null; // TODO + return null; // TODO } - /** Returns the specification of this qa run including intents to execute, specifications detailing intent translation and metrics - * to compute. */ + /** + * Returns the specification of this qa run including intents to execute, + * specifications detailing intent translation and metrics to compute. + */ public RankEvalSpec getRankEvalSpec() { return task; } - /** Returns the specification of this qa run including intents to execute, specifications detailing intent translation and metrics - * to compute. */ + /** + * Returns the specification of this qa run including intents to execute, + * specifications detailing intent translation and metrics to compute. + */ public void setRankEvalSpec(RankEvalSpec task) { this.task = task; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index f4ab789d429..7af60c24151 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -22,54 +22,65 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; import java.util.Map; -/** +/** * For each qa specification identified by its id this response returns the respective * averaged precisionAnN value. - * + * * In addition for each query the document ids that haven't been found annotated is returned as well. - * + * * Documents of unknown quality - i.e. those that haven't been supplied in the set of annotated documents but have been returned * by the search are not taken into consideration when computing precision at n - they are ignored. - * + * **/ -public class RankEvalResponse extends ActionResponse { +public class RankEvalResponse extends ActionResponse implements ToXContent { - private Collection qualityResults = new ArrayList<>(); + private RankEvalResult qualityResult; public RankEvalResponse() { - + } public RankEvalResponse(StreamInput in) throws IOException { - int size = in.readInt(); - qualityResults = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - qualityResults.add(new RankEvalResult(in)); - } + super.readFrom(in); + this.qualityResult = new RankEvalResult(in); } - + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeInt(qualityResults.size()); - for (RankEvalResult result : qualityResults) { - result.writeTo(out); - } - } - - public void addRankEvalResult(int specId, double quality, Map> unknownDocs) { - RankEvalResult result = new RankEvalResult(specId, quality, unknownDocs); - this.qualityResults.add(result); + qualityResult.writeTo(out); } - - public Collection getRankEvalResults() { - return qualityResults; + + public void setRankEvalResult(RankEvalResult result) { + this.qualityResult = result; + } + + public RankEvalResult getRankEvalResult() { + return qualityResult; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("rank_eval"); + builder.field("spec_id", qualityResult.getSpecId()); + builder.field("quality_level", qualityResult.getQualityLevel()); + builder.startArray("unknown_docs"); + Map> unknownDocs = qualityResult.getUnknownDocs(); + for (String key : unknownDocs.keySet()) { + builder.startObject(); + builder.field(key, unknownDocs.get(key)); + builder.endObject(); + } + builder.endArray(); + builder.endObject(); + return builder; } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java index b5bae8b6d4c..09c55b32b25 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java @@ -31,22 +31,23 @@ import java.util.Map; * For each precision at n computation the id of the search request specification used to generate search requests is returned * for reference. In addition the averaged precision and the ids of all documents returned but not found annotated is returned. * */ +// TODO do we need an extra class for this or it RankEvalResponse enough? public class RankEvalResult implements Writeable { - /**ID of specification this result was generated for.*/ - private int specId; - /**Average precision observed when issueing query intents with this spec.*/ + /**ID of QA specification this result was generated for.*/ + private String specId; + /**Average precision observed when issuing query intents with this specification.*/ private double qualityLevel; /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ - private Map> unknownDocs; + private Map> unknownDocs; @SuppressWarnings("unchecked") public RankEvalResult(StreamInput in) throws IOException { - this.specId = in.readInt(); + this.specId = in.readString(); this.qualityLevel = in.readDouble(); - this.unknownDocs = (Map>) in.readGenericValue(); + this.unknownDocs = (Map>) in.readGenericValue(); } - - public RankEvalResult(int specId, double quality, Map> unknownDocs) { + + public RankEvalResult(String specId, double quality, Map> unknownDocs) { this.specId = specId; this.qualityLevel = quality; this.unknownDocs = unknownDocs; @@ -54,12 +55,12 @@ public class RankEvalResult implements Writeable { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeInt(specId); + out.writeString(specId); out.writeDouble(qualityLevel); out.writeGenericValue(getUnknownDocs()); } - - public int getSpecId() { + + public String getSpecId() { return specId; } @@ -67,7 +68,12 @@ public class RankEvalResult implements Writeable { return qualityLevel; } - public Map> getUnknownDocs() { + public Map> getUnknownDocs() { return unknownDocs; } + + @Override + public String toString() { + return "RankEvalResult, ID :[" + specId + "], quality: " + qualityLevel + ", unknown docs: " + unknownDocs; + } } \ No newline at end of file diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index f625401be53..71f65c43882 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -35,45 +35,54 @@ import java.util.Collection; * */ public class RankEvalSpec implements Writeable { - /** Collection of query intents to check against including expected document ids.*/ - private Collection intents = new ArrayList<>(); + /** Collection of query specifications, that is e.g. search request templates to use for query translation. */ private Collection specifications = new ArrayList<>(); - /** Definition of n in precision at n */ + /** Definition of the quality metric, e.g. precision at N */ private RankedListQualityMetric eval; + /** a unique id for the whole QA task */ + private String taskId; + public RankEvalSpec() { + // TODO think if no args ctor is okay + } - public RankEvalSpec(Collection intents, Collection specs, RankedListQualityMetric metric) { - this.intents = intents; + public RankEvalSpec(String taskId, Collection specs, RankedListQualityMetric metric) { + this.taskId = taskId; this.specifications = specs; this.eval = metric; } public RankEvalSpec(StreamInput in) throws IOException { - int intentSize = in.readInt(); - intents = new ArrayList<>(intentSize); - for (int i = 0; i < intentSize; i++) { - intents.add(new RatedQuery(in)); - } int specSize = in.readInt(); specifications = new ArrayList<>(specSize); for (int i = 0; i < specSize; i++) { specifications.add(new QuerySpec(in)); } eval = in.readNamedWriteable(RankedListQualityMetric.class); // TODO add to registry + taskId = in.readString(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeInt(intents.size()); - for (RatedQuery query : intents) { - query.writeTo(out); - } out.writeInt(specifications.size()); for (QuerySpec spec : specifications) { spec.writeTo(out); } out.writeNamedWriteable(eval); + out.writeString(taskId); + } + + public void setEval(RankedListQualityMetric eval) { + this.eval = eval; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + public String getTaskId() { + return this.taskId; } /** Returns the precision at n configuration (containing level of n to consider).*/ @@ -86,16 +95,6 @@ public class RankEvalSpec implements Writeable { this.eval = config; } - /** Returns a list of search intents to evaluate. */ - public Collection getIntents() { - return intents; - } - - /** Set a list of search intents to evaluate. */ - public void setIntents(Collection intents) { - this.intents = intents; - } - /** Returns a list of intent to query translation specifications to evaluate. */ public Collection getSpecifications() { return specifications; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index 41168b10329..829be8a9bc5 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -19,15 +19,23 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.SearchHit; +import java.io.IOException; +import java.util.List; + /** * Classes implementing this interface provide a means to compute the quality of a result list * returned by some search. * * RelevancyLevel specifies the type of object determining the relevancy level of some known docid. * */ -public interface RankedListQualityMetric extends Evaluator { +public abstract class RankedListQualityMetric implements NamedWriteable { /** * Returns a single metric representing the ranking quality of a set of returned documents @@ -36,6 +44,27 @@ public interface RankedListQualityMetric extends Evaluator { * @param hits the result hits as returned by some search * @return some metric representing the quality of the result hit list wrt. to relevant doc ids. * */ - @Override - EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent); + public abstract EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs); + + public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { + RankedListQualityMetric rc; + Token token = parser.nextToken(); + if (token != XContentParser.Token.FIELD_NAME) { + throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name"); + } + String metricName = parser.currentName(); + + switch (metricName) { + case PrecisionAtN.NAME: + rc = PrecisionAtN.fromXContent(parser, context); + break; + default: + throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName); + } + if (parser.currentToken() == XContentParser.Token.END_OBJECT) { + // if we are at END_OBJECT, move to the next one... + parser.nextToken(); + } + return rc; + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java new file mode 100644 index 00000000000..d1bd99b97c4 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; + +import java.io.IOException; + +/** + * A document ID and its rating for the query QA use case. + * */ +public class RatedDocument implements Writeable { + + private final String docId; + private final int rating; + + public RatedDocument(String docId, int rating) { + this.docId = docId; + this.rating = rating; + } + + public RatedDocument(StreamInput in) throws IOException { + this.docId = in.readString(); + this.rating = in.readVInt(); + } + + public String getDocID() { + return docId; + } + + public int getRating() { + return rating; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(docId); + out.writeVInt(rating); + } + + public static RatedDocument fromXContent(XContentParser parser) throws IOException { + String id = null; + int rating = Integer.MIN_VALUE; + Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (parser.currentToken().equals(Token.FIELD_NAME)) { + if (id != null) { + throw new ParsingException(parser.getTokenLocation(), "only one document id allowed, found [{}] but already got [{}]", + parser.currentName(), id); + } + id = parser.currentName(); + } else if (parser.currentToken().equals(Token.VALUE_NUMBER)) { + rating = parser.intValue(); + } else { + throw new ParsingException(parser.getTokenLocation(), "unexpected token [{}] while parsing rated document", + token); + } + } + if (id == null) { + throw new ParsingException(parser.getTokenLocation(), "didn't find document id"); + } + return new RatedDocument(id, rating); + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedQuery.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedQuery.java deleted file mode 100644 index 5c4c7ea1e53..00000000000 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedQuery.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.rankeval; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; - -/** - * Objects of this class represent one type of user query to qa. Each query comprises a user supplied id for easer referencing, - * a set of parameters as supplied by the end user to the search application as well as a set of rated documents (ratings e.g. - * supplied by manual result tagging or some form of automated click log based process). - * */ -public class RatedQuery implements Writeable { - - private final int intentId; - private final Map intentParameters; - private final Map ratedDocuments; - - public RatedQuery( - int intentId, Map intentParameters, Map ratedDocuments) { - this.intentId = intentId; - this.intentParameters = intentParameters; - this.ratedDocuments = ratedDocuments; - } - - public RatedQuery(StreamInput in) throws IOException { - this.intentId = in.readInt(); - this.intentParameters = in.readMap(); - - int ratedDocsSize = in.readInt(); - this.ratedDocuments = new HashMap<>(ratedDocsSize); - for (int i = 0; i < ratedDocsSize; i++) { - this.ratedDocuments.put(in.readString(), in.readInt()); - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeInt(intentId); - out.writeMap(intentParameters); - out.writeInt(ratedDocuments.size()); - for(Entry entry : ratedDocuments.entrySet()) { - out.writeString(entry.getKey()); - out.writeInt(entry.getValue()); - } - } - - /** For easier referencing users are allowed to supply unique ids with each search intent they want to check for - * performance quality wise.*/ - public int getIntentId() { - return intentId; - } - - - /** - * Returns a mapping from query parameter name to real parameter - ideally as parsed from real user logs. - * */ - public Map getIntentParameters() { - return intentParameters; - } - - /** - * Returns a set of documents and their ratings as supplied by the users. - * */ - public Map getRatedDocuments() { - return ratedDocuments; - } - -} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 196fb9c5a9a..32f2438449a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -20,21 +20,29 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.support.RestActions; +import org.elasticsearch.rest.action.support.RestToXContentListener; import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.suggest.Suggesters; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -45,7 +53,9 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; * General Format: * * - { "requests": [{ + { + "spec_id": "human_readable_id", + "requests": [{ "id": "human_readable_id", "request": { ... request to check ... }, "ratings": { ... mapping from doc id to rating value ... } @@ -53,12 +63,15 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; "metric": { "... metric_name... ": { "... metric_parameter_key ...": ...metric_parameter_value... - }}} + } + } + } * * Example: * * - {"requests": [{ + {"spec_id": "huge_weight_on_location", + "requests": [{ "id": "amsterdam_query", "request": { "query": { @@ -78,6 +91,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; "3": 1, "4": 1 } + } }, { "id": "berlin_query", "request": { @@ -100,7 +114,9 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; }], "metric": { "precisionAtN": { - "size": 10}} + "size": 10 + } + } } * @@ -135,7 +151,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; "failed": 0 }, "rank_eval": [{ - "spec_id": "huge_weight_on_city", + "spec_id": "huge_weight_on_location", "quality_level": 0.4, "unknown_docs": [{ "amsterdam_query": [5, 10, 23] @@ -149,10 +165,17 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; * */ public class RestRankEvalAction extends BaseRestHandler { + private IndicesQueriesRegistry queryRegistry; + private AggregatorParsers aggregators; + private Suggesters suggesters; + @Inject public RestRankEvalAction(Settings settings, RestController controller, IndicesQueriesRegistry queryRegistry, AggregatorParsers aggParsers, Suggesters suggesters) { super(settings); + this.queryRegistry = queryRegistry; + this.aggregators = aggParsers; + this.suggesters = suggesters; controller.registerHandler(GET, "/_rank_eval", this); controller.registerHandler(POST, "/_rank_eval", this); controller.registerHandler(GET, "/{index}/_rank_eval", this); @@ -164,22 +187,47 @@ public class RestRankEvalAction extends BaseRestHandler { @Override public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws IOException { RankEvalRequest rankEvalRequest = new RankEvalRequest(); - //parseRankEvalRequest(rankEvalRequest, request, parseFieldMatcher); - //client.rankEval(rankEvalRequest, new RestStatusToXContentListener<>(channel)); - } - - public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, ParseFieldMatcher parseFieldMatcher) - throws IOException { - - String[] indices = Strings.splitStringByCommaToArray(request.param("index")); - BytesReference restContent = null; - if (restContent == null) { - if (RestActions.hasBodyContent(request)) { - restContent = RestActions.getRestContent(request); + BytesReference restContent = RestActions.hasBodyContent(request) ? RestActions.getRestContent(request) : null; + try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { + QueryParseContext parseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher); + if (restContent != null) { + parseRankEvalRequest(rankEvalRequest, request, + new RankEvalContext(parseFieldMatcher, parseContext, aggregators, suggesters)); } } - if (restContent != null) { - } + client.execute(RankEvalAction.INSTANCE, rankEvalRequest, new RestToXContentListener(channel)); + } + private static final ParseField SPECID_FIELD = new ParseField("spec_id"); + private static final ParseField METRIC_FIELD = new ParseField("metric"); + private static final ParseField REQUESTS_FIELD = new ParseField("requests"); + private static final ObjectParser PARSER = new ObjectParser<>("rank_eval", RankEvalSpec::new); + + static { + PARSER.declareString(RankEvalSpec::setTaskId, SPECID_FIELD); + PARSER.declareObject(RankEvalSpec::setEvaluator, (p, c) -> { + try { + return RankedListQualityMetric.fromXContent(p, c); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + } + } , METRIC_FIELD); + PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> { + try { + return QuerySpec.fromXContent(p, c); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + } + } , REQUESTS_FIELD); + } + + public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context) + throws IOException { + List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); + RankEvalSpec spec = PARSER.parse(context.parser(), context); + for (QuerySpec specification : spec.getSpecifications()) { + specification.setIndices(indices); + }; + rankEvalRequest.setRankEvalSpec(spec); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index f75bf3a0ebd..47110412d42 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -49,18 +49,18 @@ import java.util.Map; * Instances of this class execute a collection of search intents (read: user supplied query parameters) against a set of * possible search requests (read: search specifications, expressed as query/search request templates) and compares the result * against a set of annotated documents per search intent. - * + * * If any documents are returned that haven't been annotated the document id of those is returned per search intent. - * + * * The resulting search quality is computed in terms of precision at n and returned for each search specification for the full * set of search intents as averaged precision at n. * */ public class TransportRankEvalAction extends HandledTransportAction { - private SearchPhaseController searchPhaseController; - private TransportService transportService; - private SearchTransportService searchTransportService; - private ClusterService clusterService; - private ActionFilters actionFilters; + private SearchPhaseController searchPhaseController; + private TransportService transportService; + private SearchTransportService searchTransportService; + private ClusterService clusterService; + private ActionFilters actionFilters; @Inject public TransportRankEvalAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, @@ -75,46 +75,36 @@ public class TransportRankEvalAction extends HandledTransportAction listener) { - RankEvalResponse response = new RankEvalResponse(); RankEvalSpec qualityTask = request.getRankEvalSpec(); RankedListQualityMetric metric = qualityTask.getEvaluator(); - for (QuerySpec spec : qualityTask.getSpecifications()) { - double qualitySum = 0; - + double qualitySum = 0; + Map> unknownDocs = new HashMap>(); + Collection specifications = qualityTask.getSpecifications(); + for (QuerySpec spec : specifications) { SearchSourceBuilder specRequest = spec.getTestRequest(); - String[] indices = new String[spec.getIndices().size()]; + String[] indices = new String[spec.getIndices().size()]; spec.getIndices().toArray(indices); SearchRequest templatedRequest = new SearchRequest(indices, specRequest); + TransportSearchAction transportSearchAction = new TransportSearchAction(settings, threadPool, searchPhaseController, + transportService, searchTransportService, clusterService, actionFilters, indexNameExpressionResolver); + ActionFuture searchResponse = transportSearchAction.execute(templatedRequest); + SearchHits hits = searchResponse.actionGet().getHits(); - Map> unknownDocs = new HashMap>(); - Collection intents = qualityTask.getIntents(); - for (RatedQuery intent : intents) { - - TransportSearchAction transportSearchAction = new TransportSearchAction( - settings, - threadPool, - searchPhaseController, - transportService, - searchTransportService, - clusterService, - actionFilters, - indexNameExpressionResolver); - ActionFuture searchResponse = transportSearchAction.execute(templatedRequest); - SearchHits hits = searchResponse.actionGet().getHits(); - - EvalQueryQuality intentQuality = metric.evaluate(hits.getHits(), intent); - qualitySum += intentQuality.getQualityLevel(); - unknownDocs.put(intent.getIntentId(), intentQuality.getUnknownDocs()); - } - response.addRankEvalResult(spec.getSpecId(), qualitySum / intents.size(), unknownDocs); + EvalQueryQuality intentQuality = metric.evaluate(hits.getHits(), spec.getRatedDocs()); + qualitySum += intentQuality.getQualityLevel(); + unknownDocs.put(spec.getSpecId(), intentQuality.getUnknownDocs()); } + RankEvalResponse response = new RankEvalResponse(); + RankEvalResult result = new RankEvalResult(qualityTask.getTaskId(), qualitySum / specifications.size(), unknownDocs); + response.setRankEvalResult(result); listener.onResponse(response); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/PrecisionAtRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/PrecisionAtRequestTests.java deleted file mode 100644 index c6a9fa658d0..00000000000 --- a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/PrecisionAtRequestTests.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.quality; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.rankeval.PrecisionAtN; -import org.elasticsearch.index.rankeval.RankEvalPlugin; -import org.elasticsearch.index.rankeval.RatedQuery; -import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Before; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; - -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, transportClientRatio = 0.0) -// NORELEASE need to fix transport client use case -public class PrecisionAtRequestTests extends ESIntegTestCase { - @Override - protected Collection> transportClientPlugins() { - return pluginList(RankEvalPlugin.class); - } - - @Override - protected Collection> nodePlugins() { - return pluginList(RankEvalPlugin.class); - } - - @Before - public void setup() { - createIndex("test"); - ensureGreen(); - - client().prepareIndex("test", "testtype").setId("1") - .setSource("text", "berlin").get(); - client().prepareIndex("test", "testtype").setId("2") - .setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("3") - .setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("4") - .setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("5") - .setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("6") - .setSource("text", "amsterdam").get(); - refresh(); - } - - - public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { - // TODO turn into unit test - no need to execute the query here to fill hits object - MatchQueryBuilder query = new MatchQueryBuilder("text", "berlin"); - - SearchResponse response = client().prepareSearch().setQuery(query) - .execute().actionGet(); - - Map relevant = new HashMap<>(); - relevant.put("1", Rating.RELEVANT.ordinal()); - RatedQuery intent = new RatedQuery(0, new HashMap<>(), relevant); - SearchHit[] hits = response.getHits().getHits(); - - assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, intent).getQualityLevel(), 0.00001); - } - - public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException { - // TODO turn into unit test - no need to actually execute the query here to fill the hits object - MatchQueryBuilder query = new MatchQueryBuilder("text", "amsterdam"); - - SearchResponse response = client().prepareSearch().setQuery(query) - .execute().actionGet(); - - Map relevant = new HashMap<>(); - relevant.put("2", Rating.RELEVANT.ordinal()); - relevant.put("3", Rating.RELEVANT.ordinal()); - relevant.put("4", Rating.RELEVANT.ordinal()); - relevant.put("5", Rating.RELEVANT.ordinal()); - relevant.put("6", Rating.IRRELEVANT.ordinal()); - RatedQuery intent = new RatedQuery(0, new HashMap<>(), relevant); - SearchHit[] hits = response.getHits().getHits(); - - assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, intent).getQualityLevel(), 0.00001); - } - - public void testPrecisionJSON() { - - } - -/* public void testPrecisionAction() { - // TODO turn into REST test? - - Collection intents = new ArrayList(); - RatedQuery intentAmsterdam = new RatedQuery( - 0, - createParams("var", "amsterdam"), - createRelevant("2", "3", "4", "5")); - intents.add(intentAmsterdam); - - RatedQuery intentBerlin = new RatedQuery( - 1, - createParams("var", "berlin"), - createRelevant("1")); - intents.add(intentBerlin); - - Collection specs = new ArrayList(); - ArrayList indices = new ArrayList<>(); - indices.add("test"); - ArrayList types = new ArrayList<>(); - types.add("testtype"); - - SearchSourceBuilder source = new SearchSourceBuilder(); - QuerySpec spec = new QuerySpec(0, source, indices, types); - specs.add(spec); - - RankEvalSpec task = new RankEvalSpec(intents, specs, new PrecisionAtN(10)); - - RankEvalRequestBuilder builder = new RankEvalRequestBuilder( - client(), - RankEvalAction.INSTANCE, - new RankEvalRequest()); - builder.setRankEvalSpec(task); - - RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); - RankEvalResult result = response.getRankEvalResults().iterator().next(); - for (Entry> entry : result.getUnknownDocs().entrySet()) { - if (entry.getKey() == 0) { - assertEquals(1, entry.getValue().size()); - } else { - assertEquals(0, entry.getValue().size()); - } - } - }*/ - - private Map createRelevant(String... docs) { - Map relevant = new HashMap<>(); - for (String doc : docs) { - relevant.put(doc, Rating.RELEVANT.ordinal()); - } - return relevant; - } - - private Map createParams(String key, String value) { - Map parameters = new HashMap<>(); - parameters.put(key, value); - return parameters; - } - - } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java new file mode 100644 index 00000000000..79df86f6e56 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java @@ -0,0 +1,122 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.quality; + +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.rankeval.PrecisionAtN; +import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; +import org.elasticsearch.index.rankeval.QuerySpec; +import org.elasticsearch.index.rankeval.RankEvalAction; +import org.elasticsearch.index.rankeval.RankEvalPlugin; +import org.elasticsearch.index.rankeval.RankEvalRequest; +import org.elasticsearch.index.rankeval.RankEvalRequestBuilder; +import org.elasticsearch.index.rankeval.RankEvalResponse; +import org.elasticsearch.index.rankeval.RankEvalResult; +import org.elasticsearch.index.rankeval.RankEvalSpec; +import org.elasticsearch.index.rankeval.RatedDocument; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.ESIntegTestCase; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, transportClientRatio = 0.0) +// NORELEASE need to fix transport client use case +public class RankEvalRequestTests extends ESIntegTestCase { + @Override + protected Collection> transportClientPlugins() { + return pluginList(RankEvalPlugin.class); + } + + @Override + protected Collection> nodePlugins() { + return pluginList(RankEvalPlugin.class); + } + + @Before + public void setup() { + createIndex("test"); + ensureGreen(); + + client().prepareIndex("test", "testtype").setId("1") + .setSource("text", "berlin").get(); + client().prepareIndex("test", "testtype").setId("2") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("3") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("4") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("5") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("6") + .setSource("text", "amsterdam").get(); + refresh(); + } + + public void testPrecisionAtRequest() { + ArrayList indices = new ArrayList<>(); + indices.add("test"); + ArrayList types = new ArrayList<>(); + types.add("testtype"); + + String specId = randomAsciiOfLength(10); + List specifications = new ArrayList<>(); + SearchSourceBuilder testQuery = new SearchSourceBuilder(); + testQuery.query(new MatchAllQueryBuilder()); + specifications.add(new QuerySpec("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5"))); + specifications.add(new QuerySpec("berlin_query", testQuery, indices, types, createRelevant("1"))); + + RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10)); + + RankEvalRequestBuilder builder = new RankEvalRequestBuilder( + client(), + RankEvalAction.INSTANCE, + new RankEvalRequest()); + builder.setRankEvalSpec(task); + + RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + RankEvalResult result = response.getRankEvalResult(); + assertEquals(specId, result.getSpecId()); + assertEquals(1.0, result.getQualityLevel(), Double.MIN_VALUE); + Set>> entrySet = result.getUnknownDocs().entrySet(); + assertEquals(2, entrySet.size()); + for (Entry> entry : entrySet) { + if (entry.getKey() == "amsterdam_query") { + assertEquals(2, entry.getValue().size()); + } + if (entry.getKey() == "berlin_query") { + assertEquals(5, entry.getValue().size()); + } + } + } + + private static List createRelevant(String... docs) { + List relevant = new ArrayList<>(); + for (String doc : docs) { + relevant.add(new RatedDocument(doc, Rating.RELEVANT.ordinal())); + } + return relevant; + } + } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java new file mode 100644 index 00000000000..c123d5bbf8f --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; + +public class PrecisionAtNTests extends ESTestCase { + + public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + rated.add(new RatedDocument("0", Rating.RELEVANT.ordinal())); + SearchHit[] hits = new InternalSearchHit[1]; + hits[0] = new InternalSearchHit(0, "0", new Text("type"), Collections.emptyMap()); + assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + } + + public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + rated.add(new RatedDocument("0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("2", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("3", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("4", Rating.IRRELEVANT.ordinal())); + SearchHit[] hits = new InternalSearchHit[5]; + for (int i = 0; i < 5; i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text("type"), Collections.emptyMap()); + } + assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + } + + public void testParseFromXContent() throws IOException { + String xContent = " {\n" + + " \"size\": 10\n" + + "}"; + XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent); + PrecisionAtN precicionAt = PrecisionAtN.fromXContent(parser, () -> ParseFieldMatcher.STRICT); + assertEquals(10, precicionAt.getN()); + } +} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java new file mode 100644 index 00000000000..49ae4d0d6e4 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ParseFieldRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.suggest.Suggesters; +import org.elasticsearch.test.ESTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.List; + +public class QuerySpecTests extends ESTestCase { + + private static IndicesQueriesRegistry queriesRegistry; + private static SearchModule searchModule; + private static Suggesters suggesters; + private static AggregatorParsers aggsParsers; + + /** + * setup for the whole base test class + */ + @BeforeClass + public static void init() throws IOException { + aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"), new ParseFieldRegistry<>("aggregation_pipes")); + searchModule = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false); + queriesRegistry = searchModule.getQueryParserRegistry(); + suggesters = searchModule.getSuggesters(); + } + + @AfterClass + public static void afterClass() throws Exception { + queriesRegistry = null; + searchModule = null; + suggesters = null; + aggsParsers = null; + } + + public void testParseFromXContent() throws IOException { + String querySpecString = " {\n" + + " \"id\": \"my_qa_query\",\n" + + " \"request\": {\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " {\"match\": {\"beverage\": \"coffee\"}},\n" + + " {\"term\": {\"browser\": {\"value\": \"safari\"}}},\n" + + " {\"term\": {\"time_of_day\": {\"value\": \"morning\",\"boost\": 2}}},\n" + + " {\"term\": {\"ip_location\": {\"value\": \"ams\",\"boost\": 10}}}]}\n" + + " },\n" + + " \"size\": 10\n" + + " },\n" + + " \"ratings\": [ {\"1\": 1 }, { \"2\": 0 }, { \"3\": 1 } ]\n" + + "}"; + XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString); + QueryParseContext queryContext = new QueryParseContext(queriesRegistry, parser, ParseFieldMatcher.STRICT); + RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, + aggsParsers, suggesters); + QuerySpec specification = QuerySpec.fromXContent(parser, rankContext); + assertEquals("my_qa_query", specification.getSpecId()); + assertNotNull(specification.getTestRequest()); + List ratedDocs = specification.getRatedDocs(); + assertEquals(3, ratedDocs.size()); + assertEquals("1", ratedDocs.get(0).getDocID()); + assertEquals(1, ratedDocs.get(0).getRating()); + assertEquals("2", ratedDocs.get(1).getDocID()); + assertEquals(0, ratedDocs.get(1).getRating()); + assertEquals("3", ratedDocs.get(2).getDocID()); + assertEquals(1, ratedDocs.get(2).getRating()); + } +} diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 201efde33aa..d3487082b6a 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -5,44 +5,53 @@ index: index: foo type: bar - id: 1 + id: doc1 body: { "text": "berlin" } - do: index: index: foo type: bar - id: 2 + id: doc2 body: { "text": "amsterdam" } - + - do: index: index: foo type: bar - id: 3 + id: doc3 body: { "text": "amsterdam" } + + - do: + index: + index: foo + type: bar + id: doc4 + body: { "text": "something about amsterdam and berlin" } - do: indices.refresh: {} - do: rank_eval: - body: - requests: [ + body: { + "spec_id" : "cities_qa_queries", + "requests" : [ { - id: "amsterdam_query", - request: { query: {match : {text : "amsterdam" }}}, - ratings: { "1": 0, "2": 1, "3": 1 } - }, { - id: "berlin_query", - request: { query: { match : { text : "berlin" } }, size : 10 }, - ratings: {"1": 1} + "id": "amsterdam_query", + "request": { "query": { "match" : {"text" : "amsterdam" }}}, + "ratings": [{ "doc1": 0}, {"doc2": 1}, {"doc3": 1}] + }, + { + "id" : "berlin_query", + "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, + "ratings": [{"doc1": 1}] } - ] - metric: { precisionAtN: { size: 10}} - - - match: {quality_level: 1} - - gte: { took: 0 } - - is_false: task - - is_false: deleted + ], + "metric" : { "precisionatn": { "size": 10}} + } + - match: {rank_eval.spec_id: "cities_qa_queries"} + - match: {rank_eval.quality_level: 1} + - match: {rank_eval.unknown_docs.0.amsterdam_query: [ "doc4"]} + - match: {rank_eval.unknown_docs.1.berlin_query: [ "doc4"]} From c8d3098d3e8531e76ed584568aa70673da0bb2d2 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Thu, 21 Jul 2016 15:44:47 +0200 Subject: [PATCH 006/297] Move github review comments to TODOs --- .../java/org/elasticsearch/index/rankeval/PrecisionAtN.java | 1 + .../org/elasticsearch/index/rankeval/RankEvalResponse.java | 1 + .../java/org/elasticsearch/index/rankeval/RankEvalResult.java | 1 + .../elasticsearch/index/rankeval/RankedListQualityMetric.java | 1 + .../java/org/elasticsearch/index/rankeval/RatedDocument.java | 1 + .../org/elasticsearch/index/rankeval/RestRankEvalAction.java | 1 + .../elasticsearch/index/rankeval/TransportRankEvalAction.java | 1 + .../java/org/elasticsearch/index/rankeval/QuerySpecTests.java | 4 +++- 8 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 1d67c45dbb8..11101826ecb 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -128,6 +128,7 @@ public class PrecisionAtN extends RankedListQualityMetric { return new EvalQueryQuality(precision, unknownDocIds); } + // TODO add abstraction that also works for other metrics public enum Rating { IRRELEVANT, RELEVANT; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 7af60c24151..0e520febdb8 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -38,6 +38,7 @@ import java.util.Map; * Documents of unknown quality - i.e. those that haven't been supplied in the set of annotated documents but have been returned * by the search are not taken into consideration when computing precision at n - they are ignored. * + * TODO get rid of either this or RankEvalResult **/ public class RankEvalResponse extends ActionResponse implements ToXContent { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java index 09c55b32b25..726b3c82aa7 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java @@ -32,6 +32,7 @@ import java.util.Map; * for reference. In addition the averaged precision and the ids of all documents returned but not found annotated is returned. * */ // TODO do we need an extra class for this or it RankEvalResponse enough? +// TODO instead of just returning averages over complete results, think of other statistics, micro avg, macro avg, partial results public class RankEvalResult implements Writeable { /**ID of QA specification this result was generated for.*/ private String specId; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index 829be8a9bc5..76c4526ec0d 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -54,6 +54,7 @@ public abstract class RankedListQualityMetric implements NamedWriteable { } String metricName = parser.currentName(); + // TODO maybe switch to using a plugable registry later? switch (metricName) { case PrecisionAtN.NAME: rc = PrecisionAtN.fromXContent(parser, context); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index d1bd99b97c4..8915a309366 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -33,6 +33,7 @@ import java.io.IOException; * */ public class RatedDocument implements Writeable { + // TODO augment with index name and type name private final String docId; private final int rating; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 32f2438449a..5034e8350a2 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -192,6 +192,7 @@ public class RestRankEvalAction extends BaseRestHandler { QueryParseContext parseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher); if (restContent != null) { parseRankEvalRequest(rankEvalRequest, request, + // TODO can we get rid of aggregators parsers and suggesters? new RankEvalContext(parseFieldMatcher, parseContext, aggregators, suggesters)); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 47110412d42..2dbc44d7141 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -103,6 +103,7 @@ public class TransportRankEvalAction extends HandledTransportAction("aggregation"), new ParseFieldRegistry<>("aggregation_pipes")); - searchModule = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false); + searchModule = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false, new ArrayList<>()); queriesRegistry = searchModule.getQueryParserRegistry(); suggesters = searchModule.getSuggesters(); } @@ -63,6 +64,7 @@ public class QuerySpecTests extends ESTestCase { aggsParsers = null; } + // TODO add some sort of roundtrip testing like we have now for queries? public void testParseFromXContent() throws IOException { String querySpecString = " {\n" + " \"id\": \"my_qa_query\",\n" From ccf275af7b422f78dd86a1b57a2bd9d37eb5828c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 26 Jul 2016 16:10:47 +0200 Subject: [PATCH 007/297] Adapt to introduction of ESClientYamlSuiteTestCase --- .../org/elasticsearch/action/quality/RankEvalRestIT.java | 6 +++--- .../org/elasticsearch/index/rankeval/QuerySpecTests.java | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java index b68385017f6..5a7c92b97bc 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java @@ -22,19 +22,19 @@ package org.elasticsearch.action.quality; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.RestTestCandidate; import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class RankEvalRestIT extends ESRestTestCase { +public class RankEvalRestIT extends ESClientYamlSuiteTestCase { public RankEvalRestIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } @ParametersFactory public static Iterable parameters() throws IOException, RestTestParseException { - return ESRestTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(0, 1); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java index 49ae4d0d6e4..19f30c8b458 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -37,6 +37,8 @@ import org.junit.BeforeClass; import java.io.IOException; import java.util.List; +import static java.util.Collections.emptyList; + public class QuerySpecTests extends ESTestCase { private static IndicesQueriesRegistry queriesRegistry; @@ -50,7 +52,7 @@ public class QuerySpecTests extends ESTestCase { @BeforeClass public static void init() throws IOException { aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"), new ParseFieldRegistry<>("aggregation_pipes")); - searchModule = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false); + searchModule = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false, emptyList()); queriesRegistry = searchModule.getQueryParserRegistry(); suggesters = searchModule.getSuggesters(); } From 4162582ee894261f2c16d38e86cf7088d44d5f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 27 Jul 2016 11:39:59 +0200 Subject: [PATCH 008/297] Adapt to renaming of RestTestCandidate --- .../elasticsearch/action/quality/RankEvalRestIT.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java index 5a7c92b97bc..dc33d6ac439 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java @@ -22,19 +22,19 @@ package org.elasticsearch.action.quality; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.test.rest.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.RestTestCandidate; -import org.elasticsearch.test.rest.parser.RestTestParseException; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; public class RankEvalRestIT extends ESClientYamlSuiteTestCase { - public RankEvalRestIT(@Name("yaml") RestTestCandidate testCandidate) { + public RankEvalRestIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @ParametersFactory - public static Iterable parameters() throws IOException, RestTestParseException { + public static Iterable parameters() throws IOException, ClientYamlTestParseException { return ESClientYamlSuiteTestCase.createParameters(0, 1); } } From ad87bacf9178f39c06bb3a0b40f9ccac25bfeb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 1 Jul 2016 16:22:35 +0200 Subject: [PATCH 009/297] Add Reciprocal Rank query evaluation metric This adds a second query evaluation metric alongside precision_at. Reciprocal Rank is defined as 1/rank, where rank is the position of the first relevant document in the search result. The results are averaged across all queries across the sample of queries, according to https://en.wikipedia.org/wiki/Mean_reciprocal_rank --- .../index/rankeval/EvalQueryQuality.java | 8 +- .../rankeval/RankedListQualityMetric.java | 3 + .../index/rankeval/ReciprocalRank.java | 98 +++++++++++++++++++ .../action/quality/ReciprocalRankTests.java | 68 +++++++++++++ .../test/rank_eval/10_basic.yaml | 73 +++++++++++++- 5 files changed, 246 insertions(+), 4 deletions(-) create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/action/quality/ReciprocalRankTests.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index 54edd722126..98bc7f0d809 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -21,9 +21,11 @@ package org.elasticsearch.index.rankeval; import java.util.Collection; -/** Returned for each search specification. Summarizes the measured quality metric for this search request - * and adds the document ids found that were in the search result but not annotated in the original request. - * */ +/** + * Returned for each search specification. Summarizes the measured quality + * metric for this search request and adds the document ids found that were in + * the search result but not annotated in the original request. + */ public class EvalQueryQuality { private double qualityLevel; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index 829be8a9bc5..f36cca9cef5 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -58,6 +58,9 @@ public abstract class RankedListQualityMetric implements NamedWriteable { case PrecisionAtN.NAME: rc = PrecisionAtN.fromXContent(parser, context); break; + case ReciprocalRank.NAME: + rc = ReciprocalRank.fromXContent(parser, context); + break; default: throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java new file mode 100644 index 00000000000..31597507b27 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; +import org.elasticsearch.index.rankeval.PrecisionAtN.RatingMapping; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.naming.directory.SearchResult; + +/** + * Evaluate reciprocal rank. + * */ +public class ReciprocalRank extends RankedListQualityMetric { + + public static final String NAME = "reciprocal_rank"; + // the rank to use if the result list does not contain any relevant document + // TODO decide on better default or make configurable + private static final int RANK_IF_NOT_FOUND = Integer.MAX_VALUE; + + @Override + public String getWriteableName() { + return NAME; + } + + /** + * Compute ReciprocalRank based on provided relevant document IDs. + * @return reciprocal Rank for above {@link SearchResult} list. + **/ + @Override + public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { + Set relevantDocIds = new HashSet<>(); + Set irrelevantDocIds = new HashSet<>(); + for (RatedDocument doc : ratedDocs) { + if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + relevantDocIds.add(doc.getDocID()); + } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + irrelevantDocIds.add(doc.getDocID()); + } + } + + Collection unknownDocIds = new ArrayList(); + int firstRelevant = RANK_IF_NOT_FOUND; + boolean found = false; + for (int i = 0; i < hits.length; i++) { + String id = hits[i].getId(); + if (relevantDocIds.contains(id) && found == false) { + firstRelevant = i + 1; // add one because rank is not 0-based + found = true; + continue; + } else if (irrelevantDocIds.contains(id) == false) { + unknownDocIds.add(id); + } + } + + double reciprocalRank = 1.0d / firstRelevant; + return new EvalQueryQuality(reciprocalRank, unknownDocIds); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + } + + private static final ObjectParser PARSER = new ObjectParser<>( + "reciprocal_rank", () -> new ReciprocalRank()); + + public static ReciprocalRank fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { + return PARSER.apply(parser, matcher); + } +} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/ReciprocalRankTests.java new file mode 100644 index 00000000000..db16972e452 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/ReciprocalRankTests.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.quality; + +import org.elasticsearch.common.text.Text; +import org.elasticsearch.index.rankeval.EvalQueryQuality; +import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; +import org.elasticsearch.index.rankeval.RatedDocument; +import org.elasticsearch.index.rankeval.ReciprocalRank; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class ReciprocalRankTests extends ESTestCase { + + public void testEvaluationOneRelevantInResults() { + ReciprocalRank reciprocalRank = new ReciprocalRank(); + SearchHit[] hits = new SearchHit[10]; + for (int i = 0; i < 10; i++) { + hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + } + List ratedDocs = new ArrayList<>(); + // mark one of the ten docs relevant + int relevantAt = randomIntBetween(0, 9); + for (int i = 0; i <= 20; i++) { + if (i == relevantAt) { + ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.RELEVANT.ordinal())); + } else { + ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.IRRELEVANT.ordinal())); + } + } + + EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); + assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE); + } + + public void testEvaluationNoRelevantInResults() { + ReciprocalRank reciprocalRank = new ReciprocalRank(); + SearchHit[] hits = new SearchHit[10]; + for (int i = 0; i < 10; i++) { + hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + } + List ratedDocs = new ArrayList<>(); + EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); + assertEquals(1.0 / Integer.MAX_VALUE, evaluation.getQualityLevel(), Double.MIN_VALUE); + } +} diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index d3487082b6a..6707416101f 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -1,6 +1,12 @@ --- "Response format": - + - do: + indices.create: + index: foo + body: + settings: + index: + number_of_shards: 1 - do: index: index: foo @@ -55,3 +61,68 @@ - match: {rank_eval.quality_level: 1} - match: {rank_eval.unknown_docs.0.amsterdam_query: [ "doc4"]} - match: {rank_eval.unknown_docs.1.berlin_query: [ "doc4"]} +--- +"Reciprocal Rank": + + - do: + indices.create: + index: foo + body: + settings: + index: + number_of_shards: 1 + - do: + index: + index: foo + type: bar + id: doc1 + body: { "text": "berlin" } + + - do: + index: + index: foo + type: bar + id: doc2 + body: { "text": "amsterdam" } + + - do: + index: + index: foo + type: bar + id: doc3 + body: { "text": "amsterdam" } + + - do: + index: + index: foo + type: bar + id: doc4 + body: { "text": "something about amsterdam and berlin" } + + - do: + indices.refresh: {} + + - do: + rank_eval: + body: { + "spec_id" : "cities_qa_queries", + "requests" : [ + { + "id": "amsterdam_query", + "request": { "query": { "match" : {"text" : "amsterdam" }}}, + # doc4 should be returned in third position, so reciprocal rank is 1/3 + "ratings": [{ "doc4": 1}] + }, + { + "id" : "berlin_query", + "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, + # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 + "ratings": [{"doc4": 1}] + } + ], + "metric" : { "reciprocal_rank": {} } + } + + - match: {rank_eval.spec_id: "cities_qa_queries"} + # average is (1/3 + 1/2)/2 = 5/12 ~ 0.41666666666666663 + - match: {rank_eval.quality_level: 0.41666666666666663} From ad9f060dc7e08adedd3ac37adc79b5530b4962f2 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 27 Jul 2016 15:49:37 +0200 Subject: [PATCH 010/297] Use type information in request Adds parsing of type and actually using it in TransportRankEvalAction. Missing: A good idea how to actually test this in isolation... --- .../org/elasticsearch/index/rankeval/RestRankEvalAction.java | 3 +++ .../elasticsearch/index/rankeval/TransportRankEvalAction.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 5034e8350a2..7639baadc5a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -225,10 +225,13 @@ public class RestRankEvalAction extends BaseRestHandler { public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context) throws IOException { List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); + List types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type"))); RankEvalSpec spec = PARSER.parse(context.parser(), context); for (QuerySpec specification : spec.getSpecifications()) { specification.setIndices(indices); + specification.setTypes(types); }; + rankEvalRequest.setRankEvalSpec(spec); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 2dbc44d7141..ee01eb6f76c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -92,6 +92,9 @@ public class TransportRankEvalAction extends HandledTransportAction Date: Wed, 27 Jul 2016 16:24:41 +0200 Subject: [PATCH 011/297] Add option for maximally allowed rank to Reciprocal Rank metric --- .../index/rankeval/ReciprocalRank.java | 68 ++++++++++++++++--- .../action/quality/ReciprocalRankTests.java | 33 ++++++++- .../test/rank_eval/10_basic.yaml | 31 ++++++++- 3 files changed, 119 insertions(+), 13 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 31597507b27..4162896ef7d 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -19,7 +19,9 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; @@ -42,15 +44,51 @@ import javax.naming.directory.SearchResult; public class ReciprocalRank extends RankedListQualityMetric { public static final String NAME = "reciprocal_rank"; - // the rank to use if the result list does not contain any relevant document - // TODO decide on better default or make configurable - private static final int RANK_IF_NOT_FOUND = Integer.MAX_VALUE; + public static final int DEFAULT_MAX_ACCEPTABLE_RANK = 10; + private int maxAcceptableRank = DEFAULT_MAX_ACCEPTABLE_RANK; + + /** + * Initializes maxAcceptableRank with 10 + */ + public ReciprocalRank() { + // use defaults + } + + /** + * @param maxAcceptableRank + * maximal acceptable rank. Must be positive. + */ + public ReciprocalRank(int maxAcceptableRank) { + if (maxAcceptableRank <= 0) { + throw new IllegalArgumentException("maximal acceptable rank needs to be positive but was [" + maxAcceptableRank + "]"); + } + this.maxAcceptableRank = maxAcceptableRank; + } + + public ReciprocalRank(StreamInput in) throws IOException { + this.maxAcceptableRank = in.readInt(); + } @Override public String getWriteableName() { return NAME; } + /** + * @param maxAcceptableRank + * maximal acceptable rank. Must be positive. + */ + public void setMaxAcceptableRank(int maxAcceptableRank) { + if (maxAcceptableRank <= 0) { + throw new IllegalArgumentException("maximal acceptable rank needs to be positive but was [" + maxAcceptableRank + "]"); + } + this.maxAcceptableRank = maxAcceptableRank; + } + + public int getMaxAcceptableRank() { + return this.maxAcceptableRank; + } + /** * Compute ReciprocalRank based on provided relevant document IDs. * @return reciprocal Rank for above {@link SearchResult} list. @@ -67,31 +105,39 @@ public class ReciprocalRank extends RankedListQualityMetric { } } - Collection unknownDocIds = new ArrayList(); - int firstRelevant = RANK_IF_NOT_FOUND; + Collection unknownDocIds = new ArrayList<>(); + int firstRelevant = -1; boolean found = false; for (int i = 0; i < hits.length; i++) { String id = hits[i].getId(); - if (relevantDocIds.contains(id) && found == false) { - firstRelevant = i + 1; // add one because rank is not 0-based - found = true; - continue; - } else if (irrelevantDocIds.contains(id) == false) { + if (relevantDocIds.contains(id)) { + if (found == false && i < maxAcceptableRank) { + firstRelevant = i + 1; // add one because rank is not + // 0-based + found = true; + } + } else { unknownDocIds.add(id); } } - double reciprocalRank = 1.0d / firstRelevant; + double reciprocalRank = (firstRelevant == -1) ? 0 : 1.0d / firstRelevant; return new EvalQueryQuality(reciprocalRank, unknownDocIds); } @Override public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(maxAcceptableRank); } + private static final ParseField MAX_RANK_FIELD = new ParseField("max_acceptable_rank"); private static final ObjectParser PARSER = new ObjectParser<>( "reciprocal_rank", () -> new ReciprocalRank()); + static { + PARSER.declareInt(ReciprocalRank::setMaxAcceptableRank, MAX_RANK_FIELD); + } + public static ReciprocalRank fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/ReciprocalRankTests.java index db16972e452..d51b8074757 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/ReciprocalRankTests.java @@ -34,6 +34,37 @@ import java.util.List; public class ReciprocalRankTests extends ESTestCase { + public void testMaxAcceptableRank() { + ReciprocalRank reciprocalRank = new ReciprocalRank(); + assertEquals(ReciprocalRank.DEFAULT_MAX_ACCEPTABLE_RANK, reciprocalRank.getMaxAcceptableRank()); + + int maxRank = randomIntBetween(1, 100); + reciprocalRank.setMaxAcceptableRank(maxRank); + assertEquals(maxRank, reciprocalRank.getMaxAcceptableRank()); + + SearchHit[] hits = new SearchHit[10]; + for (int i = 0; i < 10; i++) { + hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + } + List ratedDocs = new ArrayList<>(); + int relevantAt = 5; + for (int i = 0; i < 10; i++) { + if (i == relevantAt) { + ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.RELEVANT.ordinal())); + } else { + ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.IRRELEVANT.ordinal())); + } + } + + int rankAtFirstRelevant = relevantAt + 1; + EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); + assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE); + + reciprocalRank = new ReciprocalRank(rankAtFirstRelevant - 1); + evaluation = reciprocalRank.evaluate(hits, ratedDocs); + assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); + } + public void testEvaluationOneRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); SearchHit[] hits = new SearchHit[10]; @@ -63,6 +94,6 @@ public class ReciprocalRankTests extends ESTestCase { } List ratedDocs = new ArrayList<>(); EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); - assertEquals(1.0 / Integer.MAX_VALUE, evaluation.getQualityLevel(), Double.MIN_VALUE); + assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); } } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 6707416101f..f2d6fbaf767 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -124,5 +124,34 @@ } - match: {rank_eval.spec_id: "cities_qa_queries"} - # average is (1/3 + 1/2)/2 = 5/12 ~ 0.41666666666666663 +# average is (1/3 + 1/2)/2 = 5/12 ~ 0.41666666666666663 - match: {rank_eval.quality_level: 0.41666666666666663} + + - do: + rank_eval: + body: { + "spec_id" : "cities_qa_queries", + "requests" : [ + { + "id": "amsterdam_query", + "request": { "query": { "match" : {"text" : "amsterdam" }}}, + # doc4 should be returned in third position, so reciprocal rank is 1/3 + "ratings": [{ "doc4": 1}] + }, + { + "id" : "berlin_query", + "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, + # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 + "ratings": [{"doc4": 1}] + } + ], + "metric" : { + "reciprocal_rank": { + "max_acceptable_rank" : 2 + } + } + } + + - match: {rank_eval.spec_id: "cities_qa_queries"} +# average is (0 + 1/2)/2 = 1/4 + - match: {rank_eval.quality_level: 0.25} From 34cbc10128d07742e09cf858f2b7a75e15d7e587 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Thu, 28 Jul 2016 11:27:06 +0200 Subject: [PATCH 012/297] Add index and type information to rated doc Also add roundtrip testing of the xcontent serialisation of RatedDoc --- .../index/rankeval/QuerySpec.java | 2 +- .../index/rankeval/RatedDocument.java | 115 ++++++++++++++---- .../action/quality/RankEvalRequestTests.java | 2 +- .../index/rankeval/PrecisionAtNTests.java | 12 +- .../index/rankeval/QuerySpecTests.java | 5 +- .../test/rank_eval/10_basic.yaml | 7 +- 6 files changed, 106 insertions(+), 37 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java index 2e82fd98939..b316ab8d1d2 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java @@ -159,7 +159,7 @@ public class QuerySpec implements Writeable { } , REQUEST_FIELD); PARSER.declareObjectArray(QuerySpec::setRatedDocs, (p, c) -> { try { - return RatedDocument.fromXContent(p); + return RatedDocument.fromXContent(p, c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index d1bd99b97c4..c76064d98f1 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -19,33 +19,83 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.ParsingException; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParser.Token; import java.io.IOException; +import java.util.Objects; /** * A document ID and its rating for the query QA use case. * */ -public class RatedDocument implements Writeable { +public class RatedDocument extends ToXContentToBytes implements Writeable { - private final String docId; - private final int rating; + public static final ParseField DOC_ID_FIELD = new ParseField("doc_id"); + public static final ParseField TYPE_FIELD = new ParseField("type"); + public static final ParseField INDEX_FIELD = new ParseField("index"); + public static final ParseField RATING_FIELD = new ParseField("rating"); - public RatedDocument(String docId, int rating) { + private static final ObjectParser PARSER = new ObjectParser<>("ratings", RatedDocument::new); + + static { + PARSER.declareString(RatedDocument::setIndex, INDEX_FIELD); + PARSER.declareString(RatedDocument::setType, TYPE_FIELD); + PARSER.declareString(RatedDocument::setDocId, DOC_ID_FIELD); + PARSER.declareInt(RatedDocument::setRating, RATING_FIELD); + } + + // TODO instead of docId use path to id and id itself + private String docId; + private String type; + private String index; + private int rating; + + RatedDocument() {} + + void setIndex(String index) { + this.index = index; + } + + void setType(String type) { + this.type = type; + } + + void setDocId(String docId) { + this.docId = docId; + } + + void setRating(int rating) { + this.rating = rating; + } + + public RatedDocument(String index, String type, String docId, int rating) { + this.index = index; + this.type = type; this.docId = docId; this.rating = rating; } public RatedDocument(StreamInput in) throws IOException { + this.index = in.readString(); + this.type = in.readString(); this.docId = in.readString(); this.rating = in.readVInt(); } + public String getIndex() { + return index; + } + + public String getType() { + return type; + } + public String getDocID() { return docId; } @@ -56,31 +106,44 @@ public class RatedDocument implements Writeable { @Override public void writeTo(StreamOutput out) throws IOException { + out.writeString(index); + out.writeString(type); out.writeString(docId); out.writeVInt(rating); } - public static RatedDocument fromXContent(XContentParser parser) throws IOException { - String id = null; - int rating = Integer.MIN_VALUE; - Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (parser.currentToken().equals(Token.FIELD_NAME)) { - if (id != null) { - throw new ParsingException(parser.getTokenLocation(), "only one document id allowed, found [{}] but already got [{}]", - parser.currentName(), id); - } - id = parser.currentName(); - } else if (parser.currentToken().equals(Token.VALUE_NUMBER)) { - rating = parser.intValue(); - } else { - throw new ParsingException(parser.getTokenLocation(), "unexpected token [{}] while parsing rated document", - token); - } + public static RatedDocument fromXContent(XContentParser parser, RankEvalContext context) throws IOException { + return PARSER.parse(parser, context); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(INDEX_FIELD.getPreferredName(), index); + builder.field(TYPE_FIELD.getPreferredName(), type); + builder.field(DOC_ID_FIELD.getPreferredName(), docId); + builder.field(RATING_FIELD.getPreferredName(), rating); + builder.endObject(); + return builder; + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; } - if (id == null) { - throw new ParsingException(parser.getTokenLocation(), "didn't find document id"); + if (obj == null || getClass() != obj.getClass()) { + return false; } - return new RatedDocument(id, rating); + RatedDocument other = (RatedDocument) obj; + return Objects.equals(index, other.index) && + Objects.equals(type, other.type) && + Objects.equals(docId, other.docId) && + Objects.equals(rating, other.rating); + } + + @Override + public final int hashCode() { + return Objects.hash(getClass(), index, type, docId, rating); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java index 79df86f6e56..1250b43c036 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java @@ -115,7 +115,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { private static List createRelevant(String... docs) { List relevant = new ArrayList<>(); for (String doc : docs) { - relevant.add(new RatedDocument(doc, Rating.RELEVANT.ordinal())); + relevant.add(new RatedDocument("test", "testtype", doc, Rating.RELEVANT.ordinal())); } return relevant; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index c123d5bbf8f..064918fe3c3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -38,7 +38,7 @@ public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument("0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); SearchHit[] hits = new InternalSearchHit[1]; hits[0] = new InternalSearchHit(0, "0", new Text("type"), Collections.emptyMap()); assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); @@ -46,11 +46,11 @@ public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument("0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("2", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("3", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("4", Rating.IRRELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); SearchHit[] hits = new InternalSearchHit[5]; for (int i = 0; i < 5; i++) { hits[i] = new InternalSearchHit(i, i+"", new Text("type"), Collections.emptyMap()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java index 19f30c8b458..ecfa5684825 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -79,7 +79,10 @@ public class QuerySpecTests extends ESTestCase { + " },\n" + " \"size\": 10\n" + " },\n" - + " \"ratings\": [ {\"1\": 1 }, { \"2\": 0 }, { \"3\": 1 } ]\n" + + " \"ratings\": [ " + + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"1\", \"rating\" : 1 }, " + + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"2\", \"rating\" : 0 }, " + + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"3\", \"rating\" : 1 }]\n" + "}"; XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString); QueryParseContext queryContext = new QueryParseContext(queriesRegistry, parser, ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index d3487082b6a..0b7f8d2e813 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -40,12 +40,15 @@ { "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, - "ratings": [{ "doc1": 0}, {"doc2": 1}, {"doc3": 1}] + "ratings": [ + { "index": "foo", "type": "bar", "doc_id": "doc1", "rating": 0}, + { "index": "foo", "type": "bar", "doc_id": "doc2", "rating": 1}, + { "index": "foo", "type": "bar", "doc_id": "doc3", "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, - "ratings": [{"doc1": 1}] + "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 1}] } ], "metric" : { "precisionatn": { "size": 10}} From 869e471f9f2b08bdee692eb8b33b6cac9c991e61 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Thu, 28 Jul 2016 11:42:05 +0200 Subject: [PATCH 013/297] Add missing file --- .../index/rankeval/RatedDocumentTests.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java new file mode 100644 index 00000000000..824a8c5bf27 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +public class RatedDocumentTests extends ESTestCase { + + public void testXContentParsing() throws IOException { + String index = randomAsciiOfLength(10); + String type = randomAsciiOfLength(10); + String docId = randomAsciiOfLength(10); + int rating = randomInt(); + RatedDocument testItem = new RatedDocument(index, type, docId, rating); + + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); + XContentBuilder shuffled = shuffleXContent(builder); + XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); + itemParser.nextToken(); + + RankEvalContext context = new RankEvalContext(ParseFieldMatcher.STRICT, null, null, null); + RatedDocument parsedItem = RatedDocument.fromXContent(itemParser, context); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + + } + +} From d71dc205fad6abfe76bded043103a94034bbc4ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 29 Jul 2016 14:49:33 +0200 Subject: [PATCH 014/297] Moving averaging of partial evaluation results to RankedListQualityMetric For the two current metrics Prec@ and reciprocal rank we currently average the partial results in the transport action. If other metric later need a different behaviour or want to parametrize this, this operation should be part of the metric itself, so this change moves it there. Also removing on of the two test packages, main code is also in one package only. --- .../index/rankeval/PrecisionAtN.java | 4 +--- .../index/rankeval/RankedListQualityMetric.java | 5 +++++ .../index/rankeval/TransportRankEvalAction.java | 15 +++++++++------ .../index/rankeval/PrecisionAtNTests.java | 12 ++++++++++++ .../rankeval}/RankEvalRequestTests.java | 2 +- .../rankeval/RankEvalYamlIT.java} | 6 +++--- .../rankeval}/ReciprocalRankTests.java | 17 +++++++++++++---- 7 files changed, 44 insertions(+), 17 deletions(-) rename modules/rank-eval/src/test/java/org/elasticsearch/{action/quality => index/rankeval}/RankEvalRequestTests.java (99%) rename modules/rank-eval/src/test/java/org/elasticsearch/{action/quality/RankEvalRestIT.java => index/rankeval/RankEvalYamlIT.java} (89%) rename modules/rank-eval/src/test/java/org/elasticsearch/{action/quality => index/rankeval}/ReciprocalRankTests.java (87%) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 11101826ecb..015206692db 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -111,7 +111,7 @@ public class PrecisionAtN extends RankedListQualityMetric { int good = 0; int bad = 0; - Collection unknownDocIds = new ArrayList(); + Collection unknownDocIds = new ArrayList<>(); for (int i = 0; (i < n && i < hits.length); i++) { String id = hits[i].getId(); if (relevantDocIds.contains(id)) { @@ -122,9 +122,7 @@ public class PrecisionAtN extends RankedListQualityMetric { unknownDocIds.add(id); } } - double precision = (double) good / (good + bad); - return new EvalQueryQuality(precision, unknownDocIds); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index 6b39afa522f..1bfd6e516f5 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -28,6 +28,7 @@ import org.elasticsearch.search.SearchHit; import java.io.IOException; import java.util.List; +import java.util.Vector; /** * Classes implementing this interface provide a means to compute the quality of a result list @@ -71,4 +72,8 @@ public abstract class RankedListQualityMetric implements NamedWriteable { } return rc; } + + double combine(Vector partialResults) { + return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size(); + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index ee01eb6f76c..ee978b7823b 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -44,6 +44,7 @@ import org.elasticsearch.transport.TransportService; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Vector; /** * Instances of this class execute a collection of search intents (read: user supplied query parameters) against a set of @@ -85,8 +86,9 @@ public class TransportRankEvalAction extends HandledTransportAction> unknownDocs = new HashMap>(); + Map> unknownDocs = new HashMap<>(); Collection specifications = qualityTask.getSpecifications(); + Vector partialResults = new Vector<>(specifications.size()); for (QuerySpec spec : specifications) { SearchSourceBuilder specRequest = spec.getTestRequest(); String[] indices = new String[spec.getIndices().size()]; @@ -101,13 +103,14 @@ public class TransportRankEvalAction extends HandledTransportAction searchResponse = transportSearchAction.execute(templatedRequest); SearchHits hits = searchResponse.actionGet().getHits(); - EvalQueryQuality intentQuality = metric.evaluate(hits.getHits(), spec.getRatedDocs()); - qualitySum += intentQuality.getQualityLevel(); - unknownDocs.put(spec.getSpecId(), intentQuality.getUnknownDocs()); + EvalQueryQuality queryQuality = metric.evaluate(hits.getHits(), spec.getRatedDocs()); + partialResults.addElement(queryQuality); + unknownDocs.put(spec.getSpecId(), queryQuality.getUnknownDocs()); } + RankEvalResponse response = new RankEvalResponse(); - // TODO move averaging to actual metric, also add other statistics - RankEvalResult result = new RankEvalResult(qualityTask.getTaskId(), qualitySum / specifications.size(), unknownDocs); + // TODO add other statistics like micro/macro avg? + RankEvalResult result = new RankEvalResult(qualityTask.getTaskId(), metric.combine(partialResults), unknownDocs); response.setRankEvalResult(result); listener.onResponse(response); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index c123d5bbf8f..1078c010f56 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -32,8 +32,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Vector; import java.util.concurrent.ExecutionException; +import static java.util.Collections.emptyList; + public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { @@ -66,4 +69,13 @@ public class PrecisionAtNTests extends ESTestCase { PrecisionAtN precicionAt = PrecisionAtN.fromXContent(parser, () -> ParseFieldMatcher.STRICT); assertEquals(10, precicionAt.getN()); } + + public void testCombine() { + PrecisionAtN metric = new PrecisionAtN(); + Vector partialResults = new Vector<>(3); + partialResults.add(new EvalQueryQuality(0.1, emptyList())); + partialResults.add(new EvalQueryQuality(0.2, emptyList())); + partialResults.add(new EvalQueryQuality(0.6, emptyList())); + assertEquals(0.3, metric.combine(partialResults), Double.MIN_VALUE); + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java similarity index 99% rename from modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index 79df86f6e56..bb0b993fbdc 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.quality; +package org.elasticsearch.index.rankeval; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.rankeval.PrecisionAtN; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java similarity index 89% rename from modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java index dc33d6ac439..4d01d183fc7 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.action.quality; +package org.elasticsearch.index.rankeval; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; @@ -28,8 +28,8 @@ import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; -public class RankEvalRestIT extends ESClientYamlSuiteTestCase { - public RankEvalRestIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { +public class RankEvalYamlIT extends ESClientYamlSuiteTestCase { + public RankEvalYamlIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java similarity index 87% rename from modules/rank-eval/src/test/java/org/elasticsearch/action/quality/ReciprocalRankTests.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index d51b8074757..12dd808cff7 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -17,13 +17,10 @@ * under the License. */ -package org.elasticsearch.action.quality; +package org.elasticsearch.index.rankeval; import org.elasticsearch.common.text.Text; -import org.elasticsearch.index.rankeval.EvalQueryQuality; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; -import org.elasticsearch.index.rankeval.RatedDocument; -import org.elasticsearch.index.rankeval.ReciprocalRank; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; @@ -31,6 +28,9 @@ import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Vector; + +import static java.util.Collections.emptyList; public class ReciprocalRankTests extends ESTestCase { @@ -86,6 +86,15 @@ public class ReciprocalRankTests extends ESTestCase { assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE); } + public void testCombine() { + ReciprocalRank reciprocalRank = new ReciprocalRank(); + Vector partialResults = new Vector<>(3); + partialResults.add(new EvalQueryQuality(0.5, emptyList())); + partialResults.add(new EvalQueryQuality(1.0, emptyList())); + partialResults.add(new EvalQueryQuality(0.75, emptyList())); + assertEquals(0.75, reciprocalRank.combine(partialResults), Double.MIN_VALUE); + } + public void testEvaluationNoRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); SearchHit[] hits = new SearchHit[10]; From 438893c0c7e1474d18a1aff40ef5ea4c858acf05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 4 Aug 2016 13:40:46 +0200 Subject: [PATCH 015/297] Adapt to changes to how named writables are registered --- .../index/rankeval/RankEvalPlugin.java | 15 +++++++++++++++ .../index/rankeval/TransportRankEvalAction.java | 3 --- .../index/rankeval/QuerySpecTests.java | 3 +-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index 7aed80fc481..bb59ff6051f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -21,10 +21,13 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestHandler; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -39,4 +42,16 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin { public List> getRestHandlers() { return Arrays.asList(RestRankEvalAction.class); } + + /** + * Returns parsers for {@link NamedWriteable} this plugin will use over the transport protocol. + * @see NamedWriteableRegistry + */ + @Override + public List getNamedWriteables() { + List metrics = new ArrayList<>(); + metrics.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, PrecisionAtN.NAME, PrecisionAtN::new)); + metrics.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); + return metrics; + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index ee978b7823b..7b13014e310 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -75,9 +75,6 @@ public class TransportRankEvalAction extends HandledTransportAction("aggregation"), new ParseFieldRegistry<>("aggregation_pipes")); - searchModule = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false, emptyList()); + searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); queriesRegistry = searchModule.getQueryParserRegistry(); suggesters = searchModule.getSuggesters(); } From 0578a964835c3328457965b1fe7e96e1234ead1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 8 Aug 2016 13:29:44 +0200 Subject: [PATCH 016/297] Merge RankEvalResult with Response The current response object only serves as a wrapper around the result object. This change merges the two classes into one. --- .../index/rankeval/RankEvalResponse.java | 56 +++++++++---- .../index/rankeval/RankEvalResult.java | 80 ------------------- .../rankeval/TransportRankEvalAction.java | 4 +- .../index/rankeval/RankEvalRequestTests.java | 17 +--- 4 files changed, 43 insertions(+), 114 deletions(-) delete mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 0e520febdb8..da8651e6a24 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -38,42 +38,64 @@ import java.util.Map; * Documents of unknown quality - i.e. those that haven't been supplied in the set of annotated documents but have been returned * by the search are not taken into consideration when computing precision at n - they are ignored. * - * TODO get rid of either this or RankEvalResult **/ +//TODO instead of just returning averages over complete results, think of other statistics, micro avg, macro avg, partial results public class RankEvalResponse extends ActionResponse implements ToXContent { - - private RankEvalResult qualityResult; + /**ID of QA specification this result was generated for.*/ + private String specId; + /**Average precision observed when issuing query intents with this specification.*/ + private double qualityLevel; + /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ + private Map> unknownDocs; public RankEvalResponse() { - } + @SuppressWarnings("unchecked") public RankEvalResponse(StreamInput in) throws IOException { super.readFrom(in); - this.qualityResult = new RankEvalResult(in); + this.specId = in.readString(); + this.qualityLevel = in.readDouble(); + this.unknownDocs = (Map>) in.readGenericValue(); + } + + public RankEvalResponse(String specId, double qualityLevel, Map> unknownDocs) { + this.specId = specId; + this.qualityLevel = qualityLevel; + this.unknownDocs = unknownDocs; + } + + public String getSpecId() { + return specId; + } + + public double getQualityLevel() { + return qualityLevel; + } + + public Map> getUnknownDocs() { + return unknownDocs; + } + + @Override + public String toString() { + return "RankEvalResult, ID :[" + specId + "], quality: " + qualityLevel + ", unknown docs: " + unknownDocs; } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - qualityResult.writeTo(out); - } - - public void setRankEvalResult(RankEvalResult result) { - this.qualityResult = result; - } - - public RankEvalResult getRankEvalResult() { - return qualityResult; + out.writeString(specId); + out.writeDouble(qualityLevel); + out.writeGenericValue(getUnknownDocs()); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("rank_eval"); - builder.field("spec_id", qualityResult.getSpecId()); - builder.field("quality_level", qualityResult.getQualityLevel()); + builder.field("spec_id", specId); + builder.field("quality_level", qualityLevel); builder.startArray("unknown_docs"); - Map> unknownDocs = qualityResult.getUnknownDocs(); for (String key : unknownDocs.keySet()) { builder.startObject(); builder.field(key, unknownDocs.get(key)); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java deleted file mode 100644 index 726b3c82aa7..00000000000 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.rankeval; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; - -import java.io.IOException; -import java.util.Collection; -import java.util.Map; - -/** - * For each precision at n computation the id of the search request specification used to generate search requests is returned - * for reference. In addition the averaged precision and the ids of all documents returned but not found annotated is returned. - * */ -// TODO do we need an extra class for this or it RankEvalResponse enough? -// TODO instead of just returning averages over complete results, think of other statistics, micro avg, macro avg, partial results -public class RankEvalResult implements Writeable { - /**ID of QA specification this result was generated for.*/ - private String specId; - /**Average precision observed when issuing query intents with this specification.*/ - private double qualityLevel; - /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ - private Map> unknownDocs; - - @SuppressWarnings("unchecked") - public RankEvalResult(StreamInput in) throws IOException { - this.specId = in.readString(); - this.qualityLevel = in.readDouble(); - this.unknownDocs = (Map>) in.readGenericValue(); - } - - public RankEvalResult(String specId, double quality, Map> unknownDocs) { - this.specId = specId; - this.qualityLevel = quality; - this.unknownDocs = unknownDocs; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(specId); - out.writeDouble(qualityLevel); - out.writeGenericValue(getUnknownDocs()); - } - - public String getSpecId() { - return specId; - } - - public double getQualityLevel() { - return qualityLevel; - } - - public Map> getUnknownDocs() { - return unknownDocs; - } - - @Override - public String toString() { - return "RankEvalResult, ID :[" + specId + "], quality: " + qualityLevel + ", unknown docs: " + unknownDocs; - } -} \ No newline at end of file diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 7b13014e310..f2e97068e62 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -105,10 +105,8 @@ public class TransportRankEvalAction extends HandledTransportAction>> entrySet = result.getUnknownDocs().entrySet(); + assertEquals(specId, response.getSpecId()); + assertEquals(1.0, response.getQualityLevel(), Double.MIN_VALUE); + Set>> entrySet = response.getUnknownDocs().entrySet(); assertEquals(2, entrySet.size()); for (Entry> entry : entrySet) { if (entry.getKey() == "amsterdam_query") { From cfaa62723d9bb57f9f6e8fcba7265d3f547284dc Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Mon, 8 Aug 2016 14:09:27 +0200 Subject: [PATCH 017/297] Actually use index/type in addition to docid for comparing hits. --- .../index/rankeval/EvalQueryQuality.java | 6 +- .../index/rankeval/PrecisionAtN.java | 18 +-- .../index/rankeval/RankEvalResponse.java | 2 +- .../index/rankeval/RankEvalResult.java | 8 +- .../index/rankeval/RatedDocument.java | 72 ++++------ .../index/rankeval/RatedDocumentKey.java | 130 ++++++++++++++++++ .../rankeval/TransportRankEvalAction.java | 2 +- .../action/quality/RankEvalRequestTests.java | 7 +- .../index/rankeval/PrecisionAtNTests.java | 55 ++++++-- .../index/rankeval/QuerySpecTests.java | 12 +- .../index/rankeval/RatedDocumentTests.java | 2 +- .../test/rank_eval/10_basic.yaml | 12 +- 12 files changed, 234 insertions(+), 92 deletions(-) create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index 54edd722126..3f408b03d0f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -27,14 +27,14 @@ import java.util.Collection; public class EvalQueryQuality { private double qualityLevel; - private Collection unknownDocs; + private Collection unknownDocs; - public EvalQueryQuality (double qualityLevel, Collection unknownDocs) { + public EvalQueryQuality (double qualityLevel, Collection unknownDocs) { this.qualityLevel = qualityLevel; this.unknownDocs = unknownDocs; } - public Collection getUnknownDocs() { + public Collection getUnknownDocs() { return unknownDocs; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 11101826ecb..292961e5c25 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -99,27 +99,27 @@ public class PrecisionAtN extends RankedListQualityMetric { @Override public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { - Collection relevantDocIds = new ArrayList<>(); - Collection irrelevantDocIds = new ArrayList<>(); + Collection relevantDocIds = new ArrayList<>(); + Collection irrelevantDocIds = new ArrayList<>(); for (RatedDocument doc : ratedDocs) { if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { - relevantDocIds.add(doc.getDocID()); + relevantDocIds.add(doc.getKey()); } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { - irrelevantDocIds.add(doc.getDocID()); + irrelevantDocIds.add(doc.getKey()); } } int good = 0; int bad = 0; - Collection unknownDocIds = new ArrayList(); + Collection unknownDocIds = new ArrayList(); for (int i = 0; (i < n && i < hits.length); i++) { - String id = hits[i].getId(); - if (relevantDocIds.contains(id)) { + RatedDocumentKey hitKey = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); + if (relevantDocIds.contains(hitKey)) { good++; - } else if (irrelevantDocIds.contains(id)) { + } else if (irrelevantDocIds.contains(hitKey)) { bad++; } else { - unknownDocIds.add(id); + unknownDocIds.add(hitKey); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 0e520febdb8..4017610b9ef 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -73,7 +73,7 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { builder.field("spec_id", qualityResult.getSpecId()); builder.field("quality_level", qualityResult.getQualityLevel()); builder.startArray("unknown_docs"); - Map> unknownDocs = qualityResult.getUnknownDocs(); + Map> unknownDocs = qualityResult.getUnknownDocs(); for (String key : unknownDocs.keySet()) { builder.startObject(); builder.field(key, unknownDocs.get(key)); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java index 726b3c82aa7..64c09155769 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java @@ -39,16 +39,16 @@ public class RankEvalResult implements Writeable { /**Average precision observed when issuing query intents with this specification.*/ private double qualityLevel; /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ - private Map> unknownDocs; + private Map> unknownDocs; @SuppressWarnings("unchecked") public RankEvalResult(StreamInput in) throws IOException { this.specId = in.readString(); this.qualityLevel = in.readDouble(); - this.unknownDocs = (Map>) in.readGenericValue(); + this.unknownDocs = (Map>) in.readGenericValue(); } - public RankEvalResult(String specId, double quality, Map> unknownDocs) { + public RankEvalResult(String specId, double quality, Map> unknownDocs) { this.specId = specId; this.qualityLevel = quality; this.unknownDocs = unknownDocs; @@ -69,7 +69,7 @@ public class RankEvalResult implements Writeable { return qualityLevel; } - public Map> getUnknownDocs() { + public Map> getUnknownDocs() { return unknownDocs; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index c76064d98f1..68ca993bdac 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -36,68 +37,51 @@ import java.util.Objects; * */ public class RatedDocument extends ToXContentToBytes implements Writeable { - public static final ParseField DOC_ID_FIELD = new ParseField("doc_id"); - public static final ParseField TYPE_FIELD = new ParseField("type"); - public static final ParseField INDEX_FIELD = new ParseField("index"); public static final ParseField RATING_FIELD = new ParseField("rating"); + public static final ParseField KEY_FIELD = new ParseField("key"); private static final ObjectParser PARSER = new ObjectParser<>("ratings", RatedDocument::new); static { - PARSER.declareString(RatedDocument::setIndex, INDEX_FIELD); - PARSER.declareString(RatedDocument::setType, TYPE_FIELD); - PARSER.declareString(RatedDocument::setDocId, DOC_ID_FIELD); + PARSER.declareObject(RatedDocument::setKey, (p, c) -> { + try { + return RatedDocumentKey.fromXContent(p, c); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + } + } , KEY_FIELD); PARSER.declareInt(RatedDocument::setRating, RATING_FIELD); } - // TODO instead of docId use path to id and id itself - private String docId; - private String type; - private String index; + private RatedDocumentKey key; private int rating; RatedDocument() {} - void setIndex(String index) { - this.index = index; + void setRatedDocumentKey(RatedDocumentKey key) { + this.key = key; } - void setType(String type) { - this.type = type; + void setKey(RatedDocumentKey key) { + this.key = key; } - - void setDocId(String docId) { - this.docId = docId; - } - + void setRating(int rating) { this.rating = rating; } - public RatedDocument(String index, String type, String docId, int rating) { - this.index = index; - this.type = type; - this.docId = docId; + public RatedDocument(RatedDocumentKey key, int rating) { + this.key = key; this.rating = rating; } public RatedDocument(StreamInput in) throws IOException { - this.index = in.readString(); - this.type = in.readString(); - this.docId = in.readString(); + this.key = new RatedDocumentKey(in); this.rating = in.readVInt(); } - public String getIndex() { - return index; - } - - public String getType() { - return type; - } - - public String getDocID() { - return docId; + public RatedDocumentKey getKey() { + return this.key; } public int getRating() { @@ -106,9 +90,7 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(index); - out.writeString(type); - out.writeString(docId); + this.key.writeTo(out); out.writeVInt(rating); } @@ -119,14 +101,12 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(INDEX_FIELD.getPreferredName(), index); - builder.field(TYPE_FIELD.getPreferredName(), type); - builder.field(DOC_ID_FIELD.getPreferredName(), docId); + builder.field(KEY_FIELD.getPreferredName(), key); builder.field(RATING_FIELD.getPreferredName(), rating); builder.endObject(); return builder; } - + @Override public final boolean equals(Object obj) { if (this == obj) { @@ -136,14 +116,12 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { return false; } RatedDocument other = (RatedDocument) obj; - return Objects.equals(index, other.index) && - Objects.equals(type, other.type) && - Objects.equals(docId, other.docId) && + return Objects.equals(key, other.key) && Objects.equals(rating, other.rating); } @Override public final int hashCode() { - return Objects.hash(getClass(), index, type, docId, rating); + return Objects.hash(getClass(), key, rating); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java new file mode 100644 index 00000000000..21551921c97 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -0,0 +1,130 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +public class RatedDocumentKey extends ToXContentToBytes implements Writeable { + public static final ParseField DOC_ID_FIELD = new ParseField("doc_id"); + public static final ParseField TYPE_FIELD = new ParseField("type"); + public static final ParseField INDEX_FIELD = new ParseField("index"); + + private static final ObjectParser PARSER = new ObjectParser<>("ratings", RatedDocumentKey::new); + + static { + PARSER.declareString(RatedDocumentKey::setIndex, INDEX_FIELD); + PARSER.declareString(RatedDocumentKey::setType, TYPE_FIELD); + PARSER.declareString(RatedDocumentKey::setDocId, DOC_ID_FIELD); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(INDEX_FIELD.getPreferredName(), index); + builder.field(TYPE_FIELD.getPreferredName(), type); + builder.field(DOC_ID_FIELD.getPreferredName(), docId); + builder.endObject(); + return builder; + } + + // TODO instead of docId use path to id and id itself + private String docId; + private String type; + private String index; + + public RatedDocumentKey() {} + + + void setIndex(String index) { + this.index = index; + } + + void setType(String type) { + this.type = type; + } + + void setDocId(String docId) { + this.docId = docId; + } + + public RatedDocumentKey(String index, String type, String docId) { + this.index = index; + this.type = type; + this.docId = docId; + } + + public RatedDocumentKey(StreamInput in) throws IOException { + this.index = in.readString(); + this.type = in.readString(); + this.docId = in.readString(); + } + + public String getIndex() { + return index; + } + + public String getType() { + return type; + } + + public String getDocID() { + return docId; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(index); + out.writeString(type); + out.writeString(docId); + } + + public static RatedDocumentKey fromXContent(XContentParser parser, RankEvalContext context) throws IOException { + return PARSER.parse(parser, context); + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + RatedDocumentKey other = (RatedDocumentKey) obj; + return Objects.equals(index, other.index) && + Objects.equals(type, other.type) && + Objects.equals(docId, other.docId); + } + + @Override + public final int hashCode() { + return Objects.hash(getClass(), index, type, docId); + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index ee01eb6f76c..4b1b146739f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -85,7 +85,7 @@ public class TransportRankEvalAction extends HandledTransportAction> unknownDocs = new HashMap>(); + Map> unknownDocs = new HashMap>(); Collection specifications = qualityTask.getSpecifications(); for (QuerySpec spec : specifications) { SearchSourceBuilder specRequest = spec.getTestRequest(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java index 1250b43c036..5926289d3b7 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.rankeval.RankEvalResponse; import org.elasticsearch.index.rankeval.RankEvalResult; import org.elasticsearch.index.rankeval.RankEvalSpec; import org.elasticsearch.index.rankeval.RatedDocument; +import org.elasticsearch.index.rankeval.RatedDocumentKey; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; @@ -100,9 +101,9 @@ public class RankEvalRequestTests extends ESIntegTestCase { RankEvalResult result = response.getRankEvalResult(); assertEquals(specId, result.getSpecId()); assertEquals(1.0, result.getQualityLevel(), Double.MIN_VALUE); - Set>> entrySet = result.getUnknownDocs().entrySet(); + Set>> entrySet = result.getUnknownDocs().entrySet(); assertEquals(2, entrySet.size()); - for (Entry> entry : entrySet) { + for (Entry> entry : entrySet) { if (entry.getKey() == "amsterdam_query") { assertEquals(2, entry.getValue().size()); } @@ -115,7 +116,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { private static List createRelevant(String... docs) { List relevant = new ArrayList<>(); for (String doc : docs) { - relevant.add(new RatedDocument("test", "testtype", doc, Rating.RELEVANT.ordinal())); + relevant.add(new RatedDocument(new RatedDocumentKey("test", "testtype", doc), Rating.RELEVANT.ordinal())); } return relevant; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index 064918fe3c3..9785bb2f1fa 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -23,8 +23,9 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; -import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; @@ -38,26 +39,58 @@ public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - SearchHit[] hits = new InternalSearchHit[1]; - hits[0] = new InternalSearchHit(0, "0", new Text("type"), Collections.emptyMap()); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), Rating.RELEVANT.ordinal())); + InternalSearchHit[] hits = new InternalSearchHit[1]; + hits[0] = new InternalSearchHit(0, "0", new Text("testtype"), Collections.emptyMap()); + hits[0].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); } public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); - SearchHit[] hits = new InternalSearchHit[5]; + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "1"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), Rating.IRRELEVANT.ordinal())); + InternalSearchHit[] hits = new InternalSearchHit[5]; for (int i = 0; i < 5; i++) { - hits[i] = new InternalSearchHit(i, i+"", new Text("type"), Collections.emptyMap()); + hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); } + public void testPrecisionAtFiveCorrectIndex() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + rated.add(new RatedDocument(new RatedDocumentKey("test_other", "testtype", "0"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test_other", "testtype", "1"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), Rating.IRRELEVANT.ordinal())); + InternalSearchHit[] hits = new InternalSearchHit[5]; + for (int i = 0; i < 5; i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); + } + assertEquals((double) 2 / 3, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + } + + public void testPrecisionAtFiveCorrectType() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + rated.add(new RatedDocument(new RatedDocumentKey("test", "other_type", "0"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "other_type", "1"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), Rating.IRRELEVANT.ordinal())); + InternalSearchHit[] hits = new InternalSearchHit[5]; + for (int i = 0; i < 5; i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); + } + assertEquals((double) 2 / 3, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + } + public void testParseFromXContent() throws IOException { String xContent = " {\n" + " \"size\": 10\n" diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java index d47dfb20745..4d25fce1d91 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -81,9 +81,9 @@ public class QuerySpecTests extends ESTestCase { + " \"size\": 10\n" + " },\n" + " \"ratings\": [ " - + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"1\", \"rating\" : 1 }, " - + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"2\", \"rating\" : 0 }, " - + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"3\", \"rating\" : 1 }]\n" + + " {\"key\": {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"1\"}, \"rating\" : 1 }, " + + " {\"key\": {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"2\"}, \"rating\" : 0 }, " + + " {\"key\": {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"3\"}, \"rating\" : 1 }]\n" + "}"; XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString); QueryParseContext queryContext = new QueryParseContext(queriesRegistry, parser, ParseFieldMatcher.STRICT); @@ -94,11 +94,11 @@ public class QuerySpecTests extends ESTestCase { assertNotNull(specification.getTestRequest()); List ratedDocs = specification.getRatedDocs(); assertEquals(3, ratedDocs.size()); - assertEquals("1", ratedDocs.get(0).getDocID()); + assertEquals("1", ratedDocs.get(0).getKey().getDocID()); assertEquals(1, ratedDocs.get(0).getRating()); - assertEquals("2", ratedDocs.get(1).getDocID()); + assertEquals("2", ratedDocs.get(1).getKey().getDocID()); assertEquals(0, ratedDocs.get(1).getRating()); - assertEquals("3", ratedDocs.get(2).getDocID()); + assertEquals("3", ratedDocs.get(2).getKey().getDocID()); assertEquals(1, ratedDocs.get(2).getRating()); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index 824a8c5bf27..ba0b1bece22 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -37,7 +37,7 @@ public class RatedDocumentTests extends ESTestCase { String type = randomAsciiOfLength(10); String docId = randomAsciiOfLength(10); int rating = randomInt(); - RatedDocument testItem = new RatedDocument(index, type, docId, rating); + RatedDocument testItem = new RatedDocument(new RatedDocumentKey(index, type, docId), rating); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 0b7f8d2e813..848debf3403 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -41,14 +41,14 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, "ratings": [ - { "index": "foo", "type": "bar", "doc_id": "doc1", "rating": 0}, - { "index": "foo", "type": "bar", "doc_id": "doc2", "rating": 1}, - { "index": "foo", "type": "bar", "doc_id": "doc3", "rating": 1}] + {"key": { "index": "foo", "type": "bar", "doc_id": "doc1"}, "rating": 0}, + {"key": { "index": "foo", "type": "bar", "doc_id": "doc2"}, "rating": 1}, + {"key": { "index": "foo", "type": "bar", "doc_id": "doc3"}, "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, - "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 1}] + "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc1"}, "rating": 1}] } ], "metric" : { "precisionatn": { "size": 10}} @@ -56,5 +56,5 @@ - match: {rank_eval.spec_id: "cities_qa_queries"} - match: {rank_eval.quality_level: 1} - - match: {rank_eval.unknown_docs.0.amsterdam_query: [ "doc4"]} - - match: {rank_eval.unknown_docs.1.berlin_query: [ "doc4"]} + - match: {rank_eval.unknown_docs.0.amsterdam_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} + - match: {rank_eval.unknown_docs.1.berlin_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} From b2fa7c4d96fb6ba14d708101310115329c78d655 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Mon, 8 Aug 2016 14:41:58 +0200 Subject: [PATCH 018/297] Fix merge errors. --- .../index/rankeval/PrecisionAtN.java | 2 +- .../index/rankeval/ReciprocalRank.java | 12 ++++---- .../index/rankeval/ReciprocalRankTests.java | 28 +++++++++++++------ .../test/rank_eval/10_basic.yaml | 8 +++--- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index a46a3f58eab..f97ba7c7975 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -111,7 +111,7 @@ public class PrecisionAtN extends RankedListQualityMetric { int good = 0; int bad = 0; - Collection unknownDocIds = new ArrayList(); + Collection unknownDocIds = new ArrayList<>(); for (int i = 0; (i < n && i < hits.length); i++) { RatedDocumentKey hitKey = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); if (relevantDocIds.contains(hitKey)) { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 4162896ef7d..70ec185fcbe 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -95,21 +95,21 @@ public class ReciprocalRank extends RankedListQualityMetric { **/ @Override public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { - Set relevantDocIds = new HashSet<>(); - Set irrelevantDocIds = new HashSet<>(); + Set relevantDocIds = new HashSet<>(); + Set irrelevantDocIds = new HashSet<>(); for (RatedDocument doc : ratedDocs) { if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { - relevantDocIds.add(doc.getDocID()); + relevantDocIds.add(doc.getKey()); } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { - irrelevantDocIds.add(doc.getDocID()); + irrelevantDocIds.add(doc.getKey()); } } - Collection unknownDocIds = new ArrayList<>(); + Collection unknownDocIds = new ArrayList<>(); int firstRelevant = -1; boolean found = false; for (int i = 0; i < hits.length; i++) { - String id = hits[i].getId(); + RatedDocumentKey id = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); if (relevantDocIds.contains(id)) { if (found == false && i < maxAcceptableRank) { firstRelevant = i + 1; // add one because rank is not diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 12dd808cff7..1abc671606c 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -20,8 +20,9 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.text.Text; +import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; -import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; @@ -42,17 +43,22 @@ public class ReciprocalRankTests extends ESTestCase { reciprocalRank.setMaxAcceptableRank(maxRank); assertEquals(maxRank, reciprocalRank.getMaxAcceptableRank()); - SearchHit[] hits = new SearchHit[10]; + InternalSearchHit[] hits = new InternalSearchHit[10]; for (int i = 0; i < 10; i++) { hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } List ratedDocs = new ArrayList<>(); int relevantAt = 5; for (int i = 0; i < 10; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument( + new RatedDocumentKey("test", "type", Integer.toString(i)), + Rating.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument( + new RatedDocumentKey("test", "type", Integer.toString(i)), + Rating.IRRELEVANT.ordinal())); } } @@ -67,18 +73,23 @@ public class ReciprocalRankTests extends ESTestCase { public void testEvaluationOneRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); - SearchHit[] hits = new SearchHit[10]; + InternalSearchHit[] hits = new InternalSearchHit[10]; for (int i = 0; i < 10; i++) { hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } List ratedDocs = new ArrayList<>(); // mark one of the ten docs relevant int relevantAt = randomIntBetween(0, 9); for (int i = 0; i <= 20; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument( + new RatedDocumentKey("test", "type", Integer.toString(i)), + Rating.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument( + new RatedDocumentKey("test", "type", Integer.toString(i)), + Rating.IRRELEVANT.ordinal())); } } @@ -97,9 +108,10 @@ public class ReciprocalRankTests extends ESTestCase { public void testEvaluationNoRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); - SearchHit[] hits = new SearchHit[10]; + InternalSearchHit[] hits = new InternalSearchHit[10]; for (int i = 0; i < 10; i++) { hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } List ratedDocs = new ArrayList<>(); EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 9eed83336c4..7dd41141b33 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -114,13 +114,13 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, # doc4 should be returned in third position, so reciprocal rank is 1/3 - "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": : 1}] + "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 - "ratings": [{"key": {"index": "foo", "type": "bar": "doc_id": "doc4"}, "rating": 1}] + "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] } ], "metric" : { "reciprocal_rank": {} } @@ -139,13 +139,13 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, # doc4 should be returned in third position, so reciprocal rank is 1/3 - "ratings": [{"key": {"index": "foo", "type": "bar": "doc_id": ""doc4"}, "rating": 1}] + "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 - "ratings": [{"key": {"index": "foo", "type": "bar": "doc_id": "doc4"}, "rating": 1}] + "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] } ], "metric" : { From 87e13ca8bbb6fdb409aa601d6b74cc400951004e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 8 Jul 2016 15:11:52 +0200 Subject: [PATCH 019/297] Add Discounted Cumulative Gain metric --- .../rankeval/DiscountedCumulativeGainAtN.java | 118 ++++++++++++++++++ .../rankeval/RankedListQualityMetric.java | 3 + .../DiscountedCumulativeGainAtNTests.java | 70 +++++++++++ .../rest-api-spec/test/rank_eval/20_dcg.yaml | 105 ++++++++++++++++ 4 files changed, 296 insertions(+) create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtN.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtNTests.java create mode 100644 modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtN.java new file mode 100644 index 00000000000..c1bed32952b --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtN.java @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DiscountedCumulativeGainAtN extends RankedListQualityMetric { + + /** Number of results to check against a given set of relevant results. */ + private int n; + + public static final String NAME = "dcg_at_n"; + private static final double LOG2 = Math.log(2.0); + + public DiscountedCumulativeGainAtN(StreamInput in) throws IOException { + n = in.readInt(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(n); + } + + @Override + public String getWriteableName() { + return NAME; + } + + /** + * Initialises n with 10 + * */ + public DiscountedCumulativeGainAtN() { + this.n = 10; + } + + /** + * @param n number of top results to check against a given set of relevant results. Must be positive. + */ + public DiscountedCumulativeGainAtN(int n) { + if (n <= 0) { + throw new IllegalArgumentException("number of results to check needs to be positive but was " + n); + } + this.n = n; + } + + /** + * Return number of search results to check for quality. + */ + public int getN() { + return n; + } + + @Override + public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { + Map ratedDocsById = new HashMap<>(); + for (RatedDocument doc : ratedDocs) { + ratedDocsById.put(doc.getDocID(), doc); + } + + Collection unknownDocIds = new ArrayList(); + double dcg = 0; + + for (int i = 0; (i < n && i < hits.length); i++) { + int rank = i + 1; // rank is 1-based + String id = hits[i].getId(); + RatedDocument ratedDoc = ratedDocsById.get(id); + if (ratedDoc != null) { + int rel = ratedDoc.getRating(); + dcg += (Math.pow(2, rel) - 1) / ((Math.log(rank + 1) / LOG2)); + } else { + unknownDocIds.add(id); + } + } + return new EvalQueryQuality(dcg, unknownDocIds); + } + + private static final ParseField SIZE_FIELD = new ParseField("size"); + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("dcg_at", a -> new DiscountedCumulativeGainAtN((Integer) a[0])); + + static { + PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD); + } + + public static DiscountedCumulativeGainAtN fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { + return PARSER.apply(parser, matcher); + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index 1bfd6e516f5..ba28b6f9bb0 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -63,6 +63,9 @@ public abstract class RankedListQualityMetric implements NamedWriteable { case ReciprocalRank.NAME: rc = ReciprocalRank.fromXContent(parser, context); break; + case DiscountedCumulativeGainAtN.NAME: + rc = DiscountedCumulativeGainAtN.fromXContent(parser, context); + break; default: throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtNTests.java new file mode 100644 index 00000000000..2114feaa835 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtNTests.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; + +public class DiscountedCumulativeGainAtNTests extends ESTestCase { + + /** + * Assuming the docs are ranked in the following order: + * + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * ------------------------------------------------------------------------------------------- + * 1 | 3 | 7.0 | 1.0 | 7.0 + * 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 + * 3 | 3 | 7.0 | 2.0 | 3.5 + * 4 | 0 | 0.0 | 2.321928094887362 | 0.0 + * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163 + * 6 | 2 | 3.0 | 2.807354922057604 | 1.0686215613240666 + */ + public void testDCGAtSix() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + int[] relevanceRatings = new int[] { 3, 2, 3, 0, 1, 2 }; + SearchHit[] hits = new InternalSearchHit[6]; + for (int i = 0; i < 6; i++) { + rated.add(new RatedDocument(Integer.toString(i), relevanceRatings[i])); + hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + } + assertEquals(13.84826362927298d, (new DiscountedCumulativeGainAtN(6)).evaluate(hits, rated).getQualityLevel(), 0.00001); + } + + + public void testParseFromXContent() throws IOException { + String xContent = " {\n" + + " \"size\": 8\n" + + "}"; + XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent); + DiscountedCumulativeGainAtN dcgAt = DiscountedCumulativeGainAtN.fromXContent(parser, () -> ParseFieldMatcher.STRICT); + assertEquals(8, dcgAt.getN()); + } +} diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml new file mode 100644 index 00000000000..6fb28286138 --- /dev/null +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml @@ -0,0 +1,105 @@ +--- +"Response format": + + - do: + index: + index: foo + type: bar + id: doc1 + body: { "bar": 1 } + + - do: + index: + index: foo + type: bar + id: doc2 + body: { "bar": 2 } + + - do: + index: + index: foo + type: bar + id: doc3 + body: { "bar": 3 } + + - do: + index: + index: foo + type: bar + id: doc4 + body: { "bar": 4 } + - do: + index: + index: foo + type: bar + id: doc5 + body: { "bar": 5 } + - do: + index: + index: foo + type: bar + id: doc6 + body: { "bar": 6 } + + - do: + indices.refresh: {} + + - do: + rank_eval: + body: { + "spec_id" : "dcg_qa_queries", + "requests" : [ + { + "id": "dcg_query", + "request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] }, + "ratings": [{ "doc1": 3}, {"doc2": 2}, {"doc3": 3}, {"doc4": 0}, {"doc5": 1}, {"doc6": 2}] + } + ], + "metric" : { "dcg_at_n": { "size": 6}} + } + + - match: {rank_eval.spec_id: "dcg_qa_queries"} + - match: {rank_eval.quality_level: 13.84826362927298} + +# reverse the order in which the results are returned (less relevant docs first) + + - do: + rank_eval: + body: { + "spec_id" : "dcg_qa_queries", + "requests" : [ + { + "id": "dcg_query_reverse", + "request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] }, + "ratings": [{ "doc1": 3}, {"doc2": 2}, {"doc3": 3}, {"doc4": 0}, {"doc5": 1}, {"doc6": 2}] + }, + ], + "metric" : { "dcg_at_n": { "size": 6}} + } + + - match: {rank_eval.spec_id: "dcg_qa_queries"} + - match: {rank_eval.quality_level: 10.29967439154499} + +# if we mix both, we should get the average + + - do: + rank_eval: + body: { + "spec_id" : "dcg_qa_queries", + "requests" : [ + { + "id": "dcg_query", + "request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] }, + "ratings": [{ "doc1": 3}, {"doc2": 2}, {"doc3": 3}, {"doc4": 0}, {"doc5": 1}, {"doc6": 2}] + }, + { + "id": "dcg_query_reverse", + "request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] }, + "ratings": [{ "doc1": 3}, {"doc2": 2}, {"doc3": 3}, {"doc4": 0}, {"doc5": 1}, {"doc6": 2}] + }, + ], + "metric" : { "dcg_at_n": { "size": 6}} + } + + - match: {rank_eval.spec_id: "dcg_qa_queries"} + - match: {rank_eval.quality_level: 12.073969010408984} From fa459f88ddb08e2bf18fe8b6cdc720f6dfc79a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 27 Jul 2016 15:22:14 +0200 Subject: [PATCH 020/297] Add normalization option When switched on, compute the normalized ndcg variant. --- .../rankeval/DiscountedCumulativeGainAt.java | 183 ++++++++++++++++++ .../rankeval/DiscountedCumulativeGainAtN.java | 118 ----------- .../rankeval/RankedListQualityMetric.java | 4 +- .../DiscountedCumulativeGainAtNTests.java | 70 ------- .../DiscountedCumulativeGainAtTests.java | 121 ++++++++++++ .../index/rankeval/ReciprocalRankTests.java | 14 +- 6 files changed, 316 insertions(+), 194 deletions(-) create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java delete mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtN.java delete mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtNTests.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java new file mode 100644 index 00000000000..2db310b185d --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -0,0 +1,183 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DiscountedCumulativeGainAt extends RankedListQualityMetric { + + /** rank position up to which to check results. */ + private int position; + /** If set to true, the dcg will be normalized (ndcg) */ + private boolean normalize; + /** If set to, this will be the rating for docs the user hasn't supplied an explicit rating for */ + private Integer unknownDocRating; + + public static final String NAME = "dcg_at_n"; + private static final double LOG2 = Math.log(2.0); + + public DiscountedCumulativeGainAt(StreamInput in) throws IOException { + position = in.readInt(); + normalize = in.readBoolean(); + unknownDocRating = in.readOptionalVInt(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(position); + out.writeBoolean(normalize); + out.writeOptionalVInt(unknownDocRating); + } + + @Override + public String getWriteableName() { + return NAME; + } + + /** + * Initialises position with 10 + * */ + public DiscountedCumulativeGainAt() { + this.position = 10; + } + + /** + * @param position number of top results to check against a given set of relevant results. Must be positive. + */ + public DiscountedCumulativeGainAt(int position) { + if (position <= 0) { + throw new IllegalArgumentException("number of results to check needs to be positive but was " + position); + } + this.position = position; + } + + /** + * Return number of search results to check for quality metric. + */ + public int getPosition() { + return this.position; + } + + /** + * set number of search results to check for quality metric. + */ + public void setPosition(int position) { + this.position = position; + } + + /** + * If set to true, the dcg will be normalized (ndcg) + */ + public void setNormalize(boolean normalize) { + this.normalize = normalize; + } + + /** + * check whether this metric computes only dcg or "normalized" ndcg + */ + public boolean getNormalize() { + return this.normalize; + } + + /** + * the rating for docs the user hasn't supplied an explicit rating for + */ + public void setUnknownDocRating(int unknownDocRating) { + this.unknownDocRating = unknownDocRating; + } + + /** + * check whether this metric computes only dcg or "normalized" ndcg + */ + public Integer getUnknownDocRating() { + return this.unknownDocRating; + } + + @Override + public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { + Map ratedDocsById = new HashMap<>(); + for (RatedDocument doc : ratedDocs) { + ratedDocsById.put(doc.getDocID(), doc); + } + + Collection unknownDocIds = new ArrayList<>(); + List ratings = new ArrayList<>(); + for (int i = 0; (i < position && i < hits.length); i++) { + String id = hits[i].getId(); + RatedDocument ratedDoc = ratedDocsById.get(id); + if (ratedDoc != null) { + ratings.add(ratedDoc.getRating()); + } else { + unknownDocIds.add(id); + if (unknownDocRating != null) { + ratings.add(unknownDocRating); + } + } + } + double dcg = computeDCG(ratings); + + if (normalize) { + Collections.sort(ratings, Collections.reverseOrder()); + double idcg = computeDCG(ratings); + dcg = dcg / idcg; + } + return new EvalQueryQuality(dcg, unknownDocIds); + } + + private static double computeDCG(List ratings) { + int rank = 1; + double dcg = 0; + for (int rating : ratings) { + dcg += (Math.pow(2, rating) - 1) / ((Math.log(rank + 1) / LOG2)); + rank++; + } + return dcg; + } + + private static final ParseField SIZE_FIELD = new ParseField("size"); + private static final ParseField NORMALIZE_FIELD = new ParseField("normalize"); + private static final ParseField UNKNOWN_DOC_RATING_FIELD = new ParseField("unknown_doc_rating"); + private static final ObjectParser PARSER = + new ObjectParser<>("dcg_at", () -> new DiscountedCumulativeGainAt()); + + static { + PARSER.declareInt(DiscountedCumulativeGainAt::setPosition, SIZE_FIELD); + PARSER.declareBoolean(DiscountedCumulativeGainAt::setNormalize, NORMALIZE_FIELD); + PARSER.declareInt(DiscountedCumulativeGainAt::setUnknownDocRating, UNKNOWN_DOC_RATING_FIELD); + } + + public static DiscountedCumulativeGainAt fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { + return PARSER.apply(parser, matcher); + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtN.java deleted file mode 100644 index c1bed32952b..00000000000 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtN.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.rankeval; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcherSupplier; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.SearchHit; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class DiscountedCumulativeGainAtN extends RankedListQualityMetric { - - /** Number of results to check against a given set of relevant results. */ - private int n; - - public static final String NAME = "dcg_at_n"; - private static final double LOG2 = Math.log(2.0); - - public DiscountedCumulativeGainAtN(StreamInput in) throws IOException { - n = in.readInt(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeInt(n); - } - - @Override - public String getWriteableName() { - return NAME; - } - - /** - * Initialises n with 10 - * */ - public DiscountedCumulativeGainAtN() { - this.n = 10; - } - - /** - * @param n number of top results to check against a given set of relevant results. Must be positive. - */ - public DiscountedCumulativeGainAtN(int n) { - if (n <= 0) { - throw new IllegalArgumentException("number of results to check needs to be positive but was " + n); - } - this.n = n; - } - - /** - * Return number of search results to check for quality. - */ - public int getN() { - return n; - } - - @Override - public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { - Map ratedDocsById = new HashMap<>(); - for (RatedDocument doc : ratedDocs) { - ratedDocsById.put(doc.getDocID(), doc); - } - - Collection unknownDocIds = new ArrayList(); - double dcg = 0; - - for (int i = 0; (i < n && i < hits.length); i++) { - int rank = i + 1; // rank is 1-based - String id = hits[i].getId(); - RatedDocument ratedDoc = ratedDocsById.get(id); - if (ratedDoc != null) { - int rel = ratedDoc.getRating(); - dcg += (Math.pow(2, rel) - 1) / ((Math.log(rank + 1) / LOG2)); - } else { - unknownDocIds.add(id); - } - } - return new EvalQueryQuality(dcg, unknownDocIds); - } - - private static final ParseField SIZE_FIELD = new ParseField("size"); - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("dcg_at", a -> new DiscountedCumulativeGainAtN((Integer) a[0])); - - static { - PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD); - } - - public static DiscountedCumulativeGainAtN fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { - return PARSER.apply(parser, matcher); - } -} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index ba28b6f9bb0..c346f0d0d59 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -63,8 +63,8 @@ public abstract class RankedListQualityMetric implements NamedWriteable { case ReciprocalRank.NAME: rc = ReciprocalRank.fromXContent(parser, context); break; - case DiscountedCumulativeGainAtN.NAME: - rc = DiscountedCumulativeGainAtN.fromXContent(parser, context); + case DiscountedCumulativeGainAt.NAME: + rc = DiscountedCumulativeGainAt.fromXContent(parser, context); break; default: throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtNTests.java deleted file mode 100644 index 2114feaa835..00000000000 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtNTests.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.rankeval; - -import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.text.Text; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.internal.InternalSearchHit; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ExecutionException; - -public class DiscountedCumulativeGainAtNTests extends ESTestCase { - - /** - * Assuming the docs are ranked in the following order: - * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) - * ------------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0 | 7.0 - * 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 - * 3 | 3 | 7.0 | 2.0 | 3.5 - * 4 | 0 | 0.0 | 2.321928094887362 | 0.0 - * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163 - * 6 | 2 | 3.0 | 2.807354922057604 | 1.0686215613240666 - */ - public void testDCGAtSix() throws IOException, InterruptedException, ExecutionException { - List rated = new ArrayList<>(); - int[] relevanceRatings = new int[] { 3, 2, 3, 0, 1, 2 }; - SearchHit[] hits = new InternalSearchHit[6]; - for (int i = 0; i < 6; i++) { - rated.add(new RatedDocument(Integer.toString(i), relevanceRatings[i])); - hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - } - assertEquals(13.84826362927298d, (new DiscountedCumulativeGainAtN(6)).evaluate(hits, rated).getQualityLevel(), 0.00001); - } - - - public void testParseFromXContent() throws IOException { - String xContent = " {\n" - + " \"size\": 8\n" - + "}"; - XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent); - DiscountedCumulativeGainAtN dcgAt = DiscountedCumulativeGainAtN.fromXContent(parser, () -> ParseFieldMatcher.STRICT); - assertEquals(8, dcgAt.getN()); - } -} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java new file mode 100644 index 00000000000..59b05a35ade --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java @@ -0,0 +1,121 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; + +public class DiscountedCumulativeGainAtTests extends ESTestCase { + + /** + * Assuming the docs are ranked in the following order: + * + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * ------------------------------------------------------------------------------------------- + * 1 | 3 | 7.0 | 1.0 | 7.0 + * 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 + * 3 | 3 | 7.0 | 2.0 | 3.5 + * 4 | 0 | 0.0 | 2.321928094887362 | 0.0 + * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163 + * 6 | 2 | 3.0 | 2.807354922057604 | 1.0686215613240666 + * + * dcg = 13.84826362927298 (sum of last column) + */ + public void testDCGAtSix() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + int[] relevanceRatings = new int[] { 3, 2, 3, 0, 1, 2 }; + SearchHit[] hits = new InternalSearchHit[6]; + for (int i = 0; i < 6; i++) { + rated.add(new RatedDocument(Integer.toString(i), relevanceRatings[i])); + hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + } + DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(6); + assertEquals(13.84826362927298, dcg.evaluate(hits, rated).getQualityLevel(), 0.00001); + + /** + * Check with normalization: to get the maximal possible dcg, sort documents by relevance in descending order + * + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * ------------------------------------------------------------------------------------------- + * 1 | 3 | 7.0 | 1.0  | 7.0 + * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 + * 3 | 2 | 3.0 | 2.0  | 1.5 + * 4 | 2 | 3.0 | 2.321928094887362  | 1.2920296742201793 + * 5 | 1 | 1.0 | 2.584962500721156  | 0.38685280723454163 + * 6 | 0 | 0.0 | 2.807354922057604  | 0.0 + * + * idcg = 14.595390756454922 (sum of last column) + */ + dcg.setNormalize(true); + assertEquals(13.84826362927298 / 14.595390756454922, dcg.evaluate(hits, rated).getQualityLevel(), 0.00001); + } + + /** + * This tests metric when some documents in the search result don't have a rating provided by the user. + * + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * ------------------------------------------------------------------------------------------- + * 1 | 3 | 7.0 | 1.0 | 7.0 + * 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 + * 3 | 3 | 7.0 | 2.0 | 3.5 + * 4 | n/a | n/a | n/a | n/a + * 5 | n/a | n/a | n/a | n/a + * 6 | n/a | n/a | n/a | n/a + * + * dcg = 13.84826362927298 (sum of last column) + */ + public void testDCGAtSixMissingRatings() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + int[] relevanceRatings = new int[] { 3, 2, 3}; + SearchHit[] hits = new InternalSearchHit[6]; + for (int i = 0; i < 6; i++) { + if (i < relevanceRatings.length) { + rated.add(new RatedDocument(Integer.toString(i), relevanceRatings[i])); + } + hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + } + DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(6); + EvalQueryQuality result = dcg.evaluate(hits, rated); + assertEquals(12.392789260714371, result.getQualityLevel(), 0.00001); + assertEquals(3, result.getUnknownDocs().size()); + } + + public void testParseFromXContent() throws IOException { + String xContent = " {\n" + + " \"size\": 8,\n" + + " \"normalize\": true\n" + + "}"; + XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent); + DiscountedCumulativeGainAt dcgAt = DiscountedCumulativeGainAt.fromXContent(parser, () -> ParseFieldMatcher.STRICT); + assertEquals(8, dcgAt.getPosition()); + assertEquals(true, dcgAt.getNormalize()); + } +} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 12dd808cff7..e87905cb1b4 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -58,11 +58,17 @@ public class ReciprocalRankTests extends ESTestCase { int rankAtFirstRelevant = relevantAt + 1; EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); - assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE); + if (rankAtFirstRelevant <= maxRank) { + assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE); - reciprocalRank = new ReciprocalRank(rankAtFirstRelevant - 1); - evaluation = reciprocalRank.evaluate(hits, ratedDocs); - assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); + // check that if we lower maxRank by one, we don't find any result and get 0.0 quality level + reciprocalRank = new ReciprocalRank(rankAtFirstRelevant - 1); + evaluation = reciprocalRank.evaluate(hits, ratedDocs); + assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); + + } else { + assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); + } } public void testEvaluationOneRelevantInResults() { From c2bd58e13d313fc7b9ab0838726b679780c0bb01 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Tue, 9 Aug 2016 12:22:44 +0200 Subject: [PATCH 021/297] Switch to ConstructingObjectParser --- .../index/rankeval/RatedDocument.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index 68ca993bdac..8e7c1aac888 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -25,7 +25,7 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -40,24 +40,23 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { public static final ParseField RATING_FIELD = new ParseField("rating"); public static final ParseField KEY_FIELD = new ParseField("key"); - private static final ObjectParser PARSER = new ObjectParser<>("ratings", RatedDocument::new); - + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rated_document", + a -> new RatedDocument((RatedDocumentKey) a[0], (Integer) a[1])); + static { - PARSER.declareObject(RatedDocument::setKey, (p, c) -> { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { try { return RatedDocumentKey.fromXContent(p, c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } } , KEY_FIELD); - PARSER.declareInt(RatedDocument::setRating, RATING_FIELD); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), RATING_FIELD); } private RatedDocumentKey key; private int rating; - RatedDocument() {} - void setRatedDocumentKey(RatedDocumentKey key) { this.key = key; } @@ -95,7 +94,7 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { } public static RatedDocument fromXContent(XContentParser parser, RankEvalContext context) throws IOException { - return PARSER.parse(parser, context); + return PARSER.apply(parser, context); } @Override From 0be997232a468e2a15a24e0f3d5b95514b8cd5d1 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Tue, 9 Aug 2016 12:49:07 +0200 Subject: [PATCH 022/297] Add ConstructingObjectParser to RatedDocumentKey --- .../index/rankeval/RatedDocumentKey.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java index 21551921c97..d68149a2ca0 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -24,7 +24,7 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -36,14 +36,15 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField INDEX_FIELD = new ParseField("index"); - private static final ObjectParser PARSER = new ObjectParser<>("ratings", RatedDocumentKey::new); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("ratings", + a -> new RatedDocumentKey((String) a[0], (String) a[1], (String) a[2])); static { - PARSER.declareString(RatedDocumentKey::setIndex, INDEX_FIELD); - PARSER.declareString(RatedDocumentKey::setType, TYPE_FIELD); - PARSER.declareString(RatedDocumentKey::setDocId, DOC_ID_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), DOC_ID_FIELD); } - + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -53,15 +54,12 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { builder.endObject(); return builder; } - + // TODO instead of docId use path to id and id itself private String docId; private String type; private String index; - public RatedDocumentKey() {} - - void setIndex(String index) { this.index = index; } @@ -106,7 +104,7 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { } public static RatedDocumentKey fromXContent(XContentParser parser, RankEvalContext context) throws IOException { - return PARSER.parse(parser, context); + return PARSER.apply(parser, context); } @Override From e71e29b3a05544526237d0347b41c66e964db3a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 2 Aug 2016 19:34:43 +0200 Subject: [PATCH 023/297] Using client instead of TransportSearchAction --- .../index/rankeval/TransportRankEvalAction.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index dc2574edf26..2748d52cb14 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.HandledTransportAction; @@ -62,6 +61,7 @@ public class TransportRankEvalAction extends HandledTransportAction> unknownDocs = new HashMap<>(); Collection specifications = qualityTask.getSpecifications(); Vector partialResults = new Vector<>(specifications.size()); @@ -95,10 +95,8 @@ public class TransportRankEvalAction extends HandledTransportAction searchResponse = transportSearchAction.execute(templatedRequest); - SearchHits hits = searchResponse.actionGet().getHits(); + ActionFuture response = client.search(templatedRequest); + SearchHits hits = response.actionGet().getHits(); EvalQueryQuality queryQuality = metric.evaluate(hits.getHits(), spec.getRatedDocs()); partialResults.addElement(queryQuality); From 795017ddfa602b78c1452a19fbc66165b14b605a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 4 Aug 2016 16:11:57 +0200 Subject: [PATCH 024/297] Adding listeners for search requests that collect results --- .../rankeval/RankedListQualityMetric.java | 4 +- .../rankeval/TransportRankEvalAction.java | 89 +++++++++++-------- 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index c346f0d0d59..86c7ebf38be 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -27,8 +27,8 @@ import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.SearchHit; import java.io.IOException; +import java.util.Collection; import java.util.List; -import java.util.Vector; /** * Classes implementing this interface provide a means to compute the quality of a result list @@ -76,7 +76,7 @@ public abstract class RankedListQualityMetric implements NamedWriteable { return rc; } - double combine(Vector partialResults) { + double combine(Collection partialResults) { return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size(); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 2748d52cb14..a68d1b14674 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -19,31 +19,25 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.action.SearchTransportService; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.controller.SearchPhaseController; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.util.Collection; import java.util.HashMap; import java.util.Map; -import java.util.Vector; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; /** * Instances of this class execute a collection of search intents (read: user supplied query parameters) against a set of @@ -56,25 +50,13 @@ import java.util.Vector; * set of search intents as averaged precision at n. * */ public class TransportRankEvalAction extends HandledTransportAction { - private SearchPhaseController searchPhaseController; - private TransportService transportService; - private SearchTransportService searchTransportService; - private ClusterService clusterService; - private ActionFilters actionFilters; private Client client; @Inject public TransportRankEvalAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, ScriptService scriptService, - AutoCreateIndex autoCreateIndex, Client client, TransportService transportService, SearchPhaseController searchPhaseController, - SearchTransportService searchTransportService, NamedWriteableRegistry namedWriteableRegistry) { + IndexNameExpressionResolver indexNameExpressionResolver, Client client, TransportService transportService) { super(settings, RankEvalAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, RankEvalRequest::new); - this.searchPhaseController = searchPhaseController; - this.transportService = transportService; - this.searchTransportService = searchTransportService; - this.clusterService = clusterService; - this.actionFilters = actionFilters; this.client = client; } @@ -85,26 +67,61 @@ public class TransportRankEvalAction extends HandledTransportAction> unknownDocs = new HashMap<>(); Collection specifications = qualityTask.getSpecifications(); - Vector partialResults = new Vector<>(specifications.size()); - for (QuerySpec spec : specifications) { - SearchSourceBuilder specRequest = spec.getTestRequest(); - String[] indices = new String[spec.getIndices().size()]; - spec.getIndices().toArray(indices); + AtomicInteger numberOfEvaluationQueries = new AtomicInteger(specifications.size()); + Map partialResults = new ConcurrentHashMap<>(specifications.size()); + + for (QuerySpec querySpecification : specifications) { + final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask, querySpecification, + partialResults, unknownDocs, numberOfEvaluationQueries); + SearchSourceBuilder specRequest = querySpecification.getTestRequest(); + String[] indices = new String[querySpecification.getIndices().size()]; + querySpecification.getIndices().toArray(indices); SearchRequest templatedRequest = new SearchRequest(indices, specRequest); - String[] types = new String[spec.getTypes().size()]; - spec.getTypes().toArray(types); + String[] types = new String[querySpecification.getTypes().size()]; + querySpecification.getTypes().toArray(types); templatedRequest.types(types); + client.search(templatedRequest, searchListener); + } + } - ActionFuture response = client.search(templatedRequest); - SearchHits hits = response.actionGet().getHits(); + public static class RankEvalActionListener implements ActionListener { - EvalQueryQuality queryQuality = metric.evaluate(hits.getHits(), spec.getRatedDocs()); - partialResults.addElement(queryQuality); - unknownDocs.put(spec.getSpecId(), queryQuality.getUnknownDocs()); + private ActionListener listener; + private QuerySpec specification; + private Map partialResults; + private RankEvalSpec task; + private Map> unknownDocs; + private AtomicInteger responseCounter; + + public RankEvalActionListener(ActionListener listener, RankEvalSpec task, QuerySpec specification, + Map partialResults, Map> unknownDocs, + AtomicInteger responseCounter) { + this.listener = listener; + this.task = task; + this.specification = specification; + this.partialResults = partialResults; + this.unknownDocs = unknownDocs; + this.responseCounter = responseCounter; } - // TODO add other statistics like micro/macro avg? - RankEvalResponse response = new RankEvalResponse(qualityTask.getTaskId(), metric.combine(partialResults), unknownDocs); - listener.onResponse(response); + @Override + public void onResponse(SearchResponse searchResponse) { + SearchHits hits = searchResponse.getHits(); + EvalQueryQuality queryQuality = task.getEvaluator().evaluate(hits.getHits(), specification.getRatedDocs()); + partialResults.put(specification.getSpecId(), queryQuality); + unknownDocs.put(specification.getSpecId(), queryQuality.getUnknownDocs()); + + if (responseCounter.decrementAndGet() == 0) { + // TODO add other statistics like micro/macro avg? + listener.onResponse( + new RankEvalResponse(task.getTaskId(), task.getEvaluator().combine(partialResults.values()), unknownDocs)); + } + } + + @Override + public void onFailure(Exception exception) { + // TODO this fails the complete request. Investigate if maybe we want to collect errors and still return partial result. + this.listener.onFailure(exception); + } } } From 8856565b895dcdb1607447f955051f268b329380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 5 Aug 2016 13:48:46 +0200 Subject: [PATCH 025/297] Make RankEvalRequestTests work with transport client --- .../index/rankeval/RankEvalAction.java | 4 +-- .../index/rankeval/RankEvalResponse.java | 25 +++++++++++-------- .../index/rankeval/RankEvalSpec.java | 2 +- .../index/rankeval/RankEvalRequestTests.java | 3 +-- .../org/elasticsearch/test/TestCluster.java | 1 + 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java index 0f506112d65..b095e948fa9 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java @@ -22,8 +22,8 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.Action; import org.elasticsearch.client.ElasticsearchClient; -/** - * Action used to start precision at qa evaluations. +/** + * Action used to start precision at qa evaluations. **/ public class RankEvalAction extends Action { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 140d60024cb..173357b9311 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -51,12 +51,8 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { public RankEvalResponse() { } - @SuppressWarnings("unchecked") public RankEvalResponse(StreamInput in) throws IOException { - super.readFrom(in); - this.specId = in.readString(); - this.qualityLevel = in.readDouble(); - this.unknownDocs = (Map>) in.readGenericValue(); + this.readFrom(in); } public RankEvalResponse(String specId, double qualityLevel, Map> unknownDocs) { @@ -65,7 +61,8 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { this.unknownDocs = unknownDocs; } - public String getSpecId() { + + public Object getSpecId() { return specId; } @@ -79,17 +76,26 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { @Override public String toString() { - return "RankEvalResult, ID :[" + specId + "], quality: " + qualityLevel + ", unknown docs: " + unknownDocs; + return "RankEvalResponse, ID :[" + specId + "], quality: " + qualityLevel + ", unknown docs: " + unknownDocs; } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeString(specId); - out.writeDouble(qualityLevel); + out.writeOptionalString(specId); + out.writeOptionalDouble(qualityLevel); out.writeGenericValue(getUnknownDocs()); } + @Override + @SuppressWarnings("unchecked") + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + this.specId = in.readOptionalString(); + this.qualityLevel = in.readOptionalDouble(); + this.unknownDocs = (Map>) in.readGenericValue(); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("rank_eval"); @@ -105,5 +111,4 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { builder.endObject(); return builder; } - } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 71f65c43882..8f3f27a792e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -59,7 +59,7 @@ public class RankEvalSpec implements Writeable { for (int i = 0; i < specSize; i++) { specifications.add(new QuerySpec(in)); } - eval = in.readNamedWriteable(RankedListQualityMetric.class); // TODO add to registry + eval = in.readNamedWriteable(RankedListQualityMetric.class); taskId = in.readString(); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index 862ccea63f3..f2b4ee65c52 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -41,8 +41,7 @@ import java.util.List; import java.util.Map.Entry; import java.util.Set; -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, transportClientRatio = 0.0) -// NORELEASE need to fix transport client use case +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) public class RankEvalRequestTests extends ESIntegTestCase { @Override protected Collection> transportClientPlugins() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java index 2629f655c95..c668c2719b6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java @@ -20,6 +20,7 @@ package org.elasticsearch.test; import com.carrotsearch.hppc.ObjectArrayList; + import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.client.Client; From a2e6dc2750adffafa13fdd59eea3cd77edbbf1c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 9 Aug 2016 18:50:15 +0200 Subject: [PATCH 026/297] Adressing review comments and adding test for failed request --- .../index/rankeval/RankEvalAction.java | 1 - .../index/rankeval/RankEvalResponse.java | 16 +++---- .../index/rankeval/ReciprocalRank.java | 1 + .../rankeval/TransportRankEvalAction.java | 8 ++-- .../index/rankeval/RankEvalRequestTests.java | 43 ++++++++++++++----- 5 files changed, 43 insertions(+), 26 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java index b095e948fa9..b8ff4b08563 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java @@ -43,5 +43,4 @@ public class RankEvalAction extends Action> unknownDocs) { this.specId = specId; this.qualityLevel = qualityLevel; @@ -62,7 +58,7 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { } - public Object getSpecId() { + public String getSpecId() { return specId; } @@ -82,8 +78,8 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeOptionalString(specId); - out.writeOptionalDouble(qualityLevel); + out.writeString(specId); + out.writeDouble(qualityLevel); out.writeGenericValue(getUnknownDocs()); } @@ -91,9 +87,9 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { @SuppressWarnings("unchecked") public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - this.specId = in.readOptionalString(); - this.qualityLevel = in.readOptionalDouble(); - this.unknownDocs = (Map>) in.readGenericValue(); + this.specId = in.readString(); + this.qualityLevel = in.readDouble(); + this.unknownDocs = (Map>) in.readGenericValue(); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 70ec185fcbe..5f23c662ea1 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -109,6 +109,7 @@ public class ReciprocalRank extends RankedListQualityMetric { int firstRelevant = -1; boolean found = false; for (int i = 0; i < hits.length; i++) { + // TODO here we use index/type/id triple not for a rated document but an unrated document in the search hits. Maybe rename? RatedDocumentKey id = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); if (relevantDocIds.contains(id)) { if (found == false && i < maxAcceptableRank) { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index a68d1b14674..0de68b2c1b9 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -63,16 +63,15 @@ public class TransportRankEvalAction extends HandledTransportAction listener) { RankEvalSpec qualityTask = request.getRankEvalSpec(); - RankedListQualityMetric metric = qualityTask.getEvaluator(); Map> unknownDocs = new HashMap<>(); Collection specifications = qualityTask.getSpecifications(); - AtomicInteger numberOfEvaluationQueries = new AtomicInteger(specifications.size()); + AtomicInteger responseCounter = new AtomicInteger(specifications.size()); Map partialResults = new ConcurrentHashMap<>(specifications.size()); for (QuerySpec querySpecification : specifications) { final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask, querySpecification, - partialResults, unknownDocs, numberOfEvaluationQueries); + partialResults, unknownDocs, responseCounter); SearchSourceBuilder specRequest = querySpecification.getTestRequest(); String[] indices = new String[querySpecification.getIndices().size()]; querySpecification.getIndices().toArray(indices); @@ -111,7 +110,7 @@ public class TransportRankEvalAction extends HandledTransportAction indices = new ArrayList<>(); - indices.add("test"); - ArrayList types = new ArrayList<>(); - types.add("testtype"); + List indices = Arrays.asList(new String[] { "test" }); + List types = Arrays.asList(new String[] { "testtype" }); String specId = randomAsciiOfLength(10); List specifications = new ArrayList<>(); SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - specifications.add(new QuerySpec("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5"))); - specifications.add(new QuerySpec("berlin_query", testQuery, indices, types, createRelevant("1"))); + specifications.add(new QuerySpec("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5"))); + specifications.add(new QuerySpec("berlin_query", testQuery, indices, types, createRelevant("1"))); RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10)); - RankEvalRequestBuilder builder = new RankEvalRequestBuilder( - client(), - RankEvalAction.INSTANCE, - new RankEvalRequest()); + RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); @@ -109,6 +107,31 @@ public class RankEvalRequestTests extends ESIntegTestCase { } } + /** + * test that running a bad query (e.g. one that will target a non existing field) will error + */ + public void testBadQuery() { + List indices = Arrays.asList(new String[] { "test" }); + List types = Arrays.asList(new String[] { "testtype" }); + + String specId = randomAsciiOfLength(10); + List specifications = new ArrayList<>(); + SearchSourceBuilder amsterdamQuery = new SearchSourceBuilder(); + amsterdamQuery.query(new MatchAllQueryBuilder()); + specifications.add(new QuerySpec("amsterdam_query", amsterdamQuery, indices, types, createRelevant("2", "3", "4", "5"))); + SearchSourceBuilder brokenQuery = new SearchSourceBuilder(); + RangeQueryBuilder brokenRangeQuery = new RangeQueryBuilder("text").timeZone("CET"); + brokenQuery.query(brokenRangeQuery); + specifications.add(new QuerySpec("broken_query", brokenQuery, indices, types, createRelevant("1"))); + + RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10)); + + RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); + builder.setRankEvalSpec(task); + + expectThrows(SearchPhaseExecutionException.class, () -> client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet()); + } + private static List createRelevant(String... docs) { List relevant = new ArrayList<>(); for (String doc : docs) { From cac4961ef40ee35b067b4d7e032a2e8623646465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 11 Aug 2016 13:27:36 +0200 Subject: [PATCH 027/297] Fix test failure because of broken RankEvalResponse serialization The introduction of RatedDocumentKey accidentally broke the response serialization because it cannot be written using writeGenericValue(). --- .../index/rankeval/RankEvalResponse.java | 45 ++++++++++++- .../rankeval/TransportRankEvalAction.java | 3 +- .../index/rankeval/RankEvalResponseTests.java | 64 +++++++++++++++++++ 3 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 20760b28f1e..9d58d47847f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -26,8 +26,11 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.Map; +import java.util.Objects; /** * For each qa specification identified by its id this response returns the respective @@ -80,16 +83,33 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { super.writeTo(out); out.writeString(specId); out.writeDouble(qualityLevel); - out.writeGenericValue(getUnknownDocs()); + out.writeVInt(unknownDocs.size()); + for (String queryId : unknownDocs.keySet()) { + out.writeString(queryId); + Collection collection = unknownDocs.get(queryId); + out.writeVInt(collection.size()); + for (RatedDocumentKey key : collection) { + key.writeTo(out); + } + } } @Override - @SuppressWarnings("unchecked") public void readFrom(StreamInput in) throws IOException { super.readFrom(in); this.specId = in.readString(); this.qualityLevel = in.readDouble(); - this.unknownDocs = (Map>) in.readGenericValue(); + int unknownDocumentSets = in.readVInt(); + this.unknownDocs = new HashMap<>(unknownDocumentSets); + for (int i = 0; i < unknownDocumentSets; i++) { + String queryId = in.readString(); + int numberUnknownDocs = in.readVInt(); + Collection collection = new ArrayList<>(numberUnknownDocs); + for (int d = 0; d < numberUnknownDocs; d++) { + collection.add(new RatedDocumentKey(in)); + } + this.unknownDocs.put(queryId, collection); + } } @Override @@ -107,4 +127,23 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { builder.endObject(); return builder; } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + RankEvalResponse other = (RankEvalResponse) obj; + return Objects.equals(specId, other.specId) && + Objects.equals(qualityLevel, other.qualityLevel) && + Objects.equals(unknownDocs, other.unknownDocs); + } + + @Override + public final int hashCode() { + return Objects.hash(getClass(), specId, qualityLevel, unknownDocs); + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 0de68b2c1b9..3e4a8d0ea19 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -34,7 +34,6 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.util.Collection; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -64,7 +63,7 @@ public class TransportRankEvalAction extends HandledTransportAction listener) { RankEvalSpec qualityTask = request.getRankEvalSpec(); - Map> unknownDocs = new HashMap<>(); + Map> unknownDocs = new ConcurrentHashMap<>(); Collection specifications = qualityTask.getSpecifications(); AtomicInteger responseCounter = new AtomicInteger(specifications.size()); Map partialResults = new ConcurrentHashMap<>(specifications.size()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java new file mode 100644 index 00000000000..89e63b0b6f0 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -0,0 +1,64 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RankEvalResponseTests extends ESTestCase { + + private static RankEvalResponse createRandomResponse() { + Map> unknownDocs = new HashMap<>(); + int numberOfSets = randomIntBetween(0, 5); + for (int i = 0; i < numberOfSets; i++) { + List ids = new ArrayList<>(); + int numberOfUnknownDocs = randomIntBetween(0, 5); + for (int d = 0; d < numberOfUnknownDocs; d++) { + ids.add(new RatedDocumentKey(randomAsciiOfLength(5), randomAsciiOfLength(5), randomAsciiOfLength(5))); + } + unknownDocs.put(randomAsciiOfLength(5), ids); + } + return new RankEvalResponse(randomAsciiOfLengthBetween(1, 10), randomDouble(), unknownDocs ); + } + + public void testSerialization() throws IOException { + RankEvalResponse randomResponse = createRandomResponse(); + try (BytesStreamOutput output = new BytesStreamOutput()) { + randomResponse.writeTo(output); + try (StreamInput in = output.bytes().streamInput()) { + RankEvalResponse deserializedResponse = new RankEvalResponse(); + deserializedResponse.readFrom(in); + assertEquals(randomResponse, deserializedResponse); + assertEquals(randomResponse.hashCode(), deserializedResponse.hashCode()); + assertNotSame(randomResponse, deserializedResponse); + assertEquals(-1, in.read()); + } + } + } + +} From b7af7c21d1a27246379960af73a84a9caae11097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 12 Aug 2016 11:03:55 +0200 Subject: [PATCH 028/297] Adapt to changes in master --- .../index/rankeval/RankEvalRequestTests.java | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index 6bcf8676d55..c03010c73b1 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -23,15 +23,6 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; -import org.elasticsearch.index.rankeval.QuerySpec; -import org.elasticsearch.index.rankeval.RankEvalAction; -import org.elasticsearch.index.rankeval.RankEvalPlugin; -import org.elasticsearch.index.rankeval.RankEvalRequest; -import org.elasticsearch.index.rankeval.RankEvalRequestBuilder; -import org.elasticsearch.index.rankeval.RankEvalResponse; -import org.elasticsearch.index.rankeval.RankEvalSpec; -import org.elasticsearch.index.rankeval.RatedDocument; -import org.elasticsearch.index.rankeval.RatedDocumentKey; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; @@ -48,12 +39,12 @@ import java.util.Set; public class RankEvalRequestTests extends ESIntegTestCase { @Override protected Collection> transportClientPlugins() { - return pluginList(RankEvalPlugin.class); + return Arrays.asList(RankEvalPlugin.class); } @Override protected Collection> nodePlugins() { - return pluginList(RankEvalPlugin.class); + return Arrays.asList(RankEvalPlugin.class); } @Before From 1eedb4c0331aae487272a58fa59e8a353b769e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 16 Aug 2016 11:48:28 +0200 Subject: [PATCH 029/297] Resolve missing imports due to changes in master --- .../org/elasticsearch/index/rankeval/RestRankEvalAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 7639baadc5a..4f369eb4b21 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -35,8 +35,8 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.support.RestActions; -import org.elasticsearch.rest.action.support.RestToXContentListener; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.suggest.Suggesters; From 2e892185f0131b5a4f820e5b01de7cf2994fe15b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 17 Aug 2016 11:54:07 +0200 Subject: [PATCH 030/297] Adapting to introduction of SearchRequestParers on master --- .../index/rankeval/RankEvalContext.java | 14 ++++++------- .../index/rankeval/RestRankEvalAction.java | 19 ++++++----------- .../index/rankeval/QuerySpecTests.java | 21 +++++++++---------- .../index/rankeval/RatedDocumentTests.java | 2 +- 4 files changed, 23 insertions(+), 33 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java index 780585d978d..496dcbca02a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java @@ -23,30 +23,28 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.suggest.Suggesters; public class RankEvalContext implements ParseFieldMatcherSupplier { + private final SearchRequestParsers searchRequestParsers; private final ParseFieldMatcher parseFieldMatcher; - private final AggregatorParsers aggs; - private final Suggesters suggesters; private final QueryParseContext parseContext; - public RankEvalContext(ParseFieldMatcher parseFieldMatcher, QueryParseContext parseContext, AggregatorParsers aggs, - Suggesters suggesters) { + public RankEvalContext(ParseFieldMatcher parseFieldMatcher, QueryParseContext parseContext, SearchRequestParsers searchRequestParsers) { this.parseFieldMatcher = parseFieldMatcher; - this.aggs = aggs; - this.suggesters = suggesters; + this.searchRequestParsers = searchRequestParsers; this.parseContext = parseContext; } public Suggesters getSuggesters() { - return this.suggesters; + return searchRequestParsers.suggesters; } public AggregatorParsers getAggs() { - return this.aggs; + return searchRequestParsers.aggParsers; } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 4f369eb4b21..8316dc49db3 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -30,15 +30,13 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.search.aggregations.AggregatorParsers; -import org.elasticsearch.search.suggest.Suggesters; +import org.elasticsearch.search.SearchRequestParsers; import java.io.IOException; import java.util.Arrays; @@ -165,17 +163,12 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; * */ public class RestRankEvalAction extends BaseRestHandler { - private IndicesQueriesRegistry queryRegistry; - private AggregatorParsers aggregators; - private Suggesters suggesters; + private SearchRequestParsers searchRequestParsers; @Inject - public RestRankEvalAction(Settings settings, RestController controller, IndicesQueriesRegistry queryRegistry, - AggregatorParsers aggParsers, Suggesters suggesters) { + public RestRankEvalAction(Settings settings, RestController controller, SearchRequestParsers searchRequestParsers) { super(settings); - this.queryRegistry = queryRegistry; - this.aggregators = aggParsers; - this.suggesters = suggesters; + this.searchRequestParsers = searchRequestParsers; controller.registerHandler(GET, "/_rank_eval", this); controller.registerHandler(POST, "/_rank_eval", this); controller.registerHandler(GET, "/{index}/_rank_eval", this); @@ -189,11 +182,11 @@ public class RestRankEvalAction extends BaseRestHandler { RankEvalRequest rankEvalRequest = new RankEvalRequest(); BytesReference restContent = RestActions.hasBodyContent(request) ? RestActions.getRestContent(request) : null; try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { - QueryParseContext parseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher); + QueryParseContext parseContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher); if (restContent != null) { parseRankEvalRequest(rankEvalRequest, request, // TODO can we get rid of aggregators parsers and suggesters? - new RankEvalContext(parseFieldMatcher, parseContext, aggregators, suggesters)); + new RankEvalContext(parseFieldMatcher, parseContext, searchRequestParsers)); } } client.execute(RankEvalAction.INSTANCE, rankEvalRequest, new RestToXContentListener(channel)); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java index ce0c367af7f..8d77e002a12 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.test.ESTestCase; @@ -40,28 +41,26 @@ import static java.util.Collections.emptyList; public class QuerySpecTests extends ESTestCase { - private static IndicesQueriesRegistry queriesRegistry; private static SearchModule searchModule; - private static Suggesters suggesters; - private static AggregatorParsers aggsParsers; + private static SearchRequestParsers searchRequestParsers; /** * setup for the whole base test class */ @BeforeClass public static void init() throws IOException { - aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"), new ParseFieldRegistry<>("aggregation_pipes")); + AggregatorParsers aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"), + new ParseFieldRegistry<>("aggregation_pipes")); searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); - queriesRegistry = searchModule.getQueryParserRegistry(); - suggesters = searchModule.getSuggesters(); + IndicesQueriesRegistry queriesRegistry = searchModule.getQueryParserRegistry(); + Suggesters suggesters = searchModule.getSuggesters(); + searchRequestParsers = new SearchRequestParsers(queriesRegistry, aggsParsers, suggesters); } @AfterClass public static void afterClass() throws Exception { - queriesRegistry = null; searchModule = null; - suggesters = null; - aggsParsers = null; + searchRequestParsers = null; } // TODO add some sort of roundtrip testing like we have now for queries? @@ -85,9 +84,9 @@ public class QuerySpecTests extends ESTestCase { + " {\"key\": {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"3\"}, \"rating\" : 1 }]\n" + "}"; XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString); - QueryParseContext queryContext = new QueryParseContext(queriesRegistry, parser, ParseFieldMatcher.STRICT); + QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, - aggsParsers, suggesters); + searchRequestParsers); QuerySpec specification = QuerySpec.fromXContent(parser, rankContext); assertEquals("my_qa_query", specification.getSpecId()); assertNotNull(specification.getTestRequest()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index ba0b1bece22..6e48b72d38e 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -48,7 +48,7 @@ public class RatedDocumentTests extends ESTestCase { XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); itemParser.nextToken(); - RankEvalContext context = new RankEvalContext(ParseFieldMatcher.STRICT, null, null, null); + RankEvalContext context = new RankEvalContext(ParseFieldMatcher.STRICT, null, null); RatedDocument parsedItem = RatedDocument.fromXContent(itemParser, context); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); From 2f506bfe04af288e395d31ee501ae7151f49ac1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 18 Aug 2016 18:14:21 +0200 Subject: [PATCH 031/297] Add toXContent method to classes used in ranking request --- .../rankeval/DiscountedCumulativeGainAt.java | 13 ++++ .../index/rankeval/PrecisionAtN.java | 9 +++ .../index/rankeval/QuerySpec.java | 22 +++++- .../index/rankeval/RankEvalSpec.java | 67 ++++++++++++++++--- .../rankeval/RankedListQualityMetric.java | 7 +- .../index/rankeval/ReciprocalRank.java | 11 +++ .../index/rankeval/RestRankEvalAction.java | 26 +------ 7 files changed, 116 insertions(+), 39 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index ab628da46aa..e620fa63d53 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; @@ -180,4 +181,16 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { public static DiscountedCumulativeGainAt fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.field(SIZE_FIELD.getPreferredName(), this.position); + builder.field(NORMALIZE_FIELD.getPreferredName(), this.normalize); + if (unknownDocRating != null) { + builder.field(UNKNOWN_DOC_RATING_FIELD.getPreferredName(), this.unknownDocRating); + } + builder.endObject(); + return builder; + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index f97ba7c7975..ba571c576ef 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; @@ -150,4 +151,12 @@ public class PrecisionAtN extends RankedListQualityMetric { } } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.field(SIZE_FIELD.getPreferredName(), this.n); + builder.endObject(); + return builder; + } + } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java index b316ab8d1d2..92445bd4697 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java @@ -19,12 +19,14 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -38,7 +40,7 @@ import java.util.List; * * The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery. * */ -public class QuerySpec implements Writeable { +public class QuerySpec extends ToXContentToBytes implements Writeable { private String specId; private SearchSourceBuilder testRequest; @@ -65,12 +67,12 @@ public class QuerySpec implements Writeable { this.specId = in.readString(); testRequest = new SearchSourceBuilder(in); int indicesSize = in.readInt(); - indices = new ArrayList(indicesSize); + indices = new ArrayList<>(indicesSize); for (int i = 0; i < indicesSize; i++) { this.indices.add(in.readString()); } int typesSize = in.readInt(); - types = new ArrayList(typesSize); + types = new ArrayList<>(typesSize); for (int i = 0; i < typesSize; i++) { this.types.add(in.readString()); } @@ -189,4 +191,18 @@ public class QuerySpec implements Writeable { public static QuerySpec fromXContent(XContentParser parser, RankEvalContext context) throws IOException { return PARSER.parse(parser, context); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ID_FIELD.getPreferredName(), this.specId); + builder.field(REQUEST_FIELD.getPreferredName(), this.testRequest); + builder.startArray(RATINGS_FIELD.getPreferredName()); + for (RatedDocument doc : this.ratedDocs) { + doc.toXContent(builder, params); + } + builder.endArray(); + builder.endObject(); + return builder; + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 8f3f27a792e..7f88104b4eb 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -19,36 +19,42 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; /** - * This class defines a qa task including query intent and query spec. + * This class defines a ranking evaluation task including an id, a collection of queries to evaluate and the evaluation metric. * * Each QA run is based on a set of queries to send to the index and multiple QA specifications that define how to translate the query - * intents into elastic search queries. In addition it contains the quality metrics to compute. + * intents into elastic search queries. * */ -public class RankEvalSpec implements Writeable { +public class RankEvalSpec extends ToXContentToBytes implements Writeable { /** Collection of query specifications, that is e.g. search request templates to use for query translation. */ private Collection specifications = new ArrayList<>(); /** Definition of the quality metric, e.g. precision at N */ private RankedListQualityMetric eval; /** a unique id for the whole QA task */ - private String taskId; + private String specId; public RankEvalSpec() { // TODO think if no args ctor is okay } - public RankEvalSpec(String taskId, Collection specs, RankedListQualityMetric metric) { - this.taskId = taskId; + public RankEvalSpec(String specId, Collection specs, RankedListQualityMetric metric) { + this.specId = specId; this.specifications = specs; this.eval = metric; } @@ -60,7 +66,7 @@ public class RankEvalSpec implements Writeable { specifications.add(new QuerySpec(in)); } eval = in.readNamedWriteable(RankedListQualityMetric.class); - taskId = in.readString(); + specId = in.readString(); } @Override @@ -70,7 +76,7 @@ public class RankEvalSpec implements Writeable { spec.writeTo(out); } out.writeNamedWriteable(eval); - out.writeString(taskId); + out.writeString(specId); } public void setEval(RankedListQualityMetric eval) { @@ -78,11 +84,11 @@ public class RankEvalSpec implements Writeable { } public void setTaskId(String taskId) { - this.taskId = taskId; + this.specId = taskId; } public String getTaskId() { - return this.taskId; + return this.specId; } /** Returns the precision at n configuration (containing level of n to consider).*/ @@ -105,4 +111,45 @@ public class RankEvalSpec implements Writeable { this.specifications = specifications; } + private static final ParseField SPECID_FIELD = new ParseField("spec_id"); + private static final ParseField METRIC_FIELD = new ParseField("metric"); + private static final ParseField REQUESTS_FIELD = new ParseField("requests"); + private static final ObjectParser PARSER = new ObjectParser<>("rank_eval", RankEvalSpec::new); + + static { + PARSER.declareString(RankEvalSpec::setTaskId, SPECID_FIELD); + PARSER.declareObject(RankEvalSpec::setEvaluator, (p, c) -> { + try { + return RankedListQualityMetric.fromXContent(p, c); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + } + } , METRIC_FIELD); + PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> { + try { + return QuerySpec.fromXContent(p, c); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + } + } , REQUESTS_FIELD); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(SPECID_FIELD.getPreferredName(), this.specId); + builder.startArray(REQUESTS_FIELD.getPreferredName()); + for (QuerySpec spec : this.specifications) { + spec.toXContent(builder, params); + } + builder.endArray(); + builder.field(METRIC_FIELD.getPreferredName(), this.eval); + builder.endObject(); + return builder; + } + + public static RankEvalSpec parse(XContentParser parser, RankEvalContext context) throws IOException { + return PARSER.parse(parser, context); + } + } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index 86c7ebf38be..e423ab3533c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -19,9 +19,11 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.SearchHit; @@ -36,7 +38,7 @@ import java.util.List; * * RelevancyLevel specifies the type of object determining the relevancy level of some known docid. * */ -public abstract class RankedListQualityMetric implements NamedWriteable { +public abstract class RankedListQualityMetric extends ToXContentToBytes implements NamedWriteable { /** * Returns a single metric representing the ranking quality of a set of returned documents @@ -79,4 +81,7 @@ public abstract class RankedListQualityMetric implements NamedWriteable { double combine(Collection partialResults) { return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size(); } + + @Override + public abstract XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 5f23c662ea1..dd4e710859b 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; import org.elasticsearch.index.rankeval.PrecisionAtN.RatingMapping; @@ -142,4 +143,14 @@ public class ReciprocalRank extends RankedListQualityMetric { public static ReciprocalRank fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startObject(NAME); + builder.field(MAX_RANK_FIELD.getPreferredName(), this.maxAcceptableRank); + builder.endObject(); + builder.endObject(); + return builder; + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 8316dc49db3..8acc7b34a51 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -20,13 +20,10 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; @@ -192,34 +189,13 @@ public class RestRankEvalAction extends BaseRestHandler { client.execute(RankEvalAction.INSTANCE, rankEvalRequest, new RestToXContentListener(channel)); } - private static final ParseField SPECID_FIELD = new ParseField("spec_id"); - private static final ParseField METRIC_FIELD = new ParseField("metric"); - private static final ParseField REQUESTS_FIELD = new ParseField("requests"); - private static final ObjectParser PARSER = new ObjectParser<>("rank_eval", RankEvalSpec::new); - static { - PARSER.declareString(RankEvalSpec::setTaskId, SPECID_FIELD); - PARSER.declareObject(RankEvalSpec::setEvaluator, (p, c) -> { - try { - return RankedListQualityMetric.fromXContent(p, c); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); - } - } , METRIC_FIELD); - PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> { - try { - return QuerySpec.fromXContent(p, c); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); - } - } , REQUESTS_FIELD); - } public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context) throws IOException { List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); List types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type"))); - RankEvalSpec spec = PARSER.parse(context.parser(), context); + RankEvalSpec spec = RankEvalSpec.parse(context.parser(), context); for (QuerySpec specification : spec.getSpecifications()) { specification.setIndices(indices); specification.setTypes(types); From a2a92b96297a8ac0012b4d8921b3bf4623c4043d Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 24 Aug 2016 14:21:24 +0200 Subject: [PATCH 032/297] Add roundtrip xcontent test to DiscountedCumulativeGainAt This factors the roundtripping out of RatedDocumentTests. Makes RankedListQualityMetric and RatedDocument implement FromXContenBuilder to be able to do the aforementioned refactoring in a generic way. Adds a roundtrip test to DiscountedCumulativeGainAt. Open questions: DiscountedCumulativeGain didn't have a constructor that accepted all possible parameters as arguments. Added one. I guess we still want to keep the one that only requires the position argument? To make roundtripping work I had to change the NAME parameter when generating XContent for DiscountedCumulativeGainAt - all remaining unit tests seem to be passing (haven't checked the REST tests yet) - need to figure out why that was there to begin with. --- .../rankeval/DiscountedCumulativeGainAt.java | 47 ++++++++++++++++- .../index/rankeval/PrecisionAtN.java | 14 ++++- .../rankeval/RankedListQualityMetric.java | 5 +- .../index/rankeval/RatedDocument.java | 22 ++++++-- .../index/rankeval/RatedDocumentKey.java | 5 +- .../index/rankeval/ReciprocalRank.java | 14 ++++- .../DiscountedCumulativeGainAtTests.java | 12 ++++- .../index/rankeval/RatedDocumentTests.java | 30 ++--------- .../rankeval/XContentRoundtripTestCase.java | 51 +++++++++++++++++++ 9 files changed, 160 insertions(+), 40 deletions(-) create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTestCase.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index e620fa63d53..18e79bec3b9 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -35,8 +36,9 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; -public class DiscountedCumulativeGainAt extends RankedListQualityMetric { +public class DiscountedCumulativeGainAt extends RankedListQualityMetric { /** rank position up to which to check results. */ private int position; @@ -83,6 +85,17 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { this.position = position; } + /** + * @param position number of top results to check against a given set of relevant results. Must be positive. // TODO is there a way to enforce this? + * @param normalize If set to true, dcg will be normalized (ndcg) See https://en.wikipedia.org/wiki/Discounted_cumulative_gain + * @param unknownDocRating the rating for docs the user hasn't supplied an explicit rating for + * */ + public DiscountedCumulativeGainAt(int position, boolean normalize, Integer unknownDocRating) { + this.position = position; + this.normalize = normalize; + this.unknownDocRating = unknownDocRating; + } + /** * Return number of search results to check for quality metric. */ @@ -178,13 +191,24 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { PARSER.declareInt(DiscountedCumulativeGainAt::setUnknownDocRating, UNKNOWN_DOC_RATING_FIELD); } + @Override + public DiscountedCumulativeGainAt fromXContent(XContentParser parser, ParseFieldMatcher matcher) { + return DiscountedCumulativeGainAt.fromXContent(parser, new ParseFieldMatcherSupplier() { + @Override + public ParseFieldMatcher getParseFieldMatcher() { + return matcher; + } + }); + } + public static DiscountedCumulativeGainAt fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(NAME); + //builder.startObject(NAME); // TODO roundtrip xcontent only works w/o this, wtf? + builder.startObject(); builder.field(SIZE_FIELD.getPreferredName(), this.position); builder.field(NORMALIZE_FIELD.getPreferredName(), this.normalize); if (unknownDocRating != null) { @@ -193,4 +217,23 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { builder.endObject(); return builder; } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + DiscountedCumulativeGainAt other = (DiscountedCumulativeGainAt) obj; + return Objects.equals(position, other.position) && + Objects.equals(normalize, other.normalize) && + Objects.equals(unknownDocRating, other.unknownDocRating); + } + + @Override + public final int hashCode() { + return Objects.hash(getClass(), position, normalize, unknownDocRating); + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index ba571c576ef..c9ad3deb759 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -40,7 +41,7 @@ import javax.naming.directory.SearchResult; * * Documents of unkonwn quality are ignored in the precision at n computation and returned by document id. * */ -public class PrecisionAtN extends RankedListQualityMetric { +public class PrecisionAtN extends RankedListQualityMetric { /** Number of results to check against a given set of relevant results. */ private int n; @@ -90,6 +91,17 @@ public class PrecisionAtN extends RankedListQualityMetric { PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD); } + @Override + public PrecisionAtN fromXContent(XContentParser parser, ParseFieldMatcher matcher) { + return PrecisionAtN.fromXContent(parser, new ParseFieldMatcherSupplier() { + + @Override + public ParseFieldMatcher getParseFieldMatcher() { + return matcher; + } + }); + } + public static PrecisionAtN fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index e423ab3533c..30793d564e3 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.FromXContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; @@ -38,7 +39,9 @@ import java.util.List; * * RelevancyLevel specifies the type of object determining the relevancy level of some known docid. * */ -public abstract class RankedListQualityMetric extends ToXContentToBytes implements NamedWriteable { +public abstract class RankedListQualityMetric + extends ToXContentToBytes + implements NamedWriteable, FromXContentBuilder { /** * Returns a single metric representing the ranking quality of a set of returned documents diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index 8e7c1aac888..c5faecb986e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -21,11 +21,14 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.FromXContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -35,12 +38,12 @@ import java.util.Objects; /** * A document ID and its rating for the query QA use case. * */ -public class RatedDocument extends ToXContentToBytes implements Writeable { +public class RatedDocument extends ToXContentToBytes implements Writeable, FromXContentBuilder { public static final ParseField RATING_FIELD = new ParseField("rating"); public static final ParseField KEY_FIELD = new ParseField("key"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rated_document", + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rated_document", a -> new RatedDocument((RatedDocumentKey) a[0], (Integer) a[1])); static { @@ -93,8 +96,19 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { out.writeVInt(rating); } - public static RatedDocument fromXContent(XContentParser parser, RankEvalContext context) throws IOException { - return PARSER.apply(parser, context); + @Override + public RatedDocument fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException { + return RatedDocument.fromXContent(parser, new ParseFieldMatcherSupplier() { + + @Override + public ParseFieldMatcher getParseFieldMatcher() { + return parseFieldMatcher; + } + }); + } + + public static RatedDocument fromXContent(XContentParser parser, ParseFieldMatcherSupplier supplier) throws IOException { + return PARSER.apply(parser, supplier); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java index d68149a2ca0..c10bc3f0ce0 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -36,7 +37,7 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField INDEX_FIELD = new ParseField("index"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("ratings", + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("ratings", a -> new RatedDocumentKey((String) a[0], (String) a[1], (String) a[2])); static { @@ -103,7 +104,7 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { out.writeString(docId); } - public static RatedDocumentKey fromXContent(XContentParser parser, RankEvalContext context) throws IOException { + public static RatedDocumentKey fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { return PARSER.apply(parser, context); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index dd4e710859b..791e857eca8 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -42,7 +43,7 @@ import javax.naming.directory.SearchResult; /** * Evaluate reciprocal rank. * */ -public class ReciprocalRank extends RankedListQualityMetric { +public class ReciprocalRank extends RankedListQualityMetric { public static final String NAME = "reciprocal_rank"; public static final int DEFAULT_MAX_ACCEPTABLE_RANK = 10; @@ -140,6 +141,17 @@ public class ReciprocalRank extends RankedListQualityMetric { PARSER.declareInt(ReciprocalRank::setMaxAcceptableRank, MAX_RANK_FIELD); } + @Override + public ReciprocalRank fromXContent(XContentParser parser, ParseFieldMatcher matcher) { + return ReciprocalRank.fromXContent(parser, new ParseFieldMatcherSupplier() { + + @Override + public ParseFieldMatcher getParseFieldMatcher() { + return matcher; + } + }); + } + public static ReciprocalRank fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java index 2221ee2a1e3..68207470382 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; -import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; @@ -34,7 +33,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; -public class DiscountedCumulativeGainAtTests extends ESTestCase { +public class DiscountedCumulativeGainAtTests extends XContentRoundtripTestCase { /** * Assuming the docs are ranked in the following order: @@ -121,4 +120,13 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { assertEquals(8, dcgAt.getPosition()); assertEquals(true, dcgAt.getNormalize()); } + + public void testXContentRoundtrip() throws IOException { + int position = randomIntBetween(0, 1000); + boolean normalize = randomBoolean(); + Integer unknownDocRating = new Integer(randomIntBetween(0, 1000)); + + DiscountedCumulativeGainAt testItem = new DiscountedCumulativeGainAt(position, normalize, unknownDocRating); + roundtrip(testItem); + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index 6e48b72d38e..8cb4a194026 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -19,41 +19,17 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.test.ESTestCase; - import java.io.IOException; -public class RatedDocumentTests extends ESTestCase { +public class RatedDocumentTests extends XContentRoundtripTestCase { public void testXContentParsing() throws IOException { String index = randomAsciiOfLength(10); String type = randomAsciiOfLength(10); String docId = randomAsciiOfLength(10); int rating = randomInt(); + RatedDocument testItem = new RatedDocument(new RatedDocumentKey(index, type, docId), rating); - - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - if (randomBoolean()) { - builder.prettyPrint(); - } - testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); - XContentBuilder shuffled = shuffleXContent(builder); - XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); - itemParser.nextToken(); - - RankEvalContext context = new RankEvalContext(ParseFieldMatcher.STRICT, null, null); - RatedDocument parsedItem = RatedDocument.fromXContent(itemParser, context); - assertNotSame(testItem, parsedItem); - assertEquals(testItem, parsedItem); - assertEquals(testItem.hashCode(), parsedItem.hashCode()); - + roundtrip(testItem); } - } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTestCase.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTestCase.java new file mode 100644 index 00000000000..689a42726ff --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTestCase.java @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.FromXContentBuilder; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +public class XContentRoundtripTestCase> extends ESTestCase { + + public void roundtrip(T testItem) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); + XContentBuilder shuffled = shuffleXContent(builder); + XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); + itemParser.nextToken(); + T parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + } +} From 5c9cc1d453297ad8e706f7800398e7610c70ef1f Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 24 Aug 2016 14:39:12 +0200 Subject: [PATCH 033/297] Add roundtripping to PrecisionAtN --- .../index/rankeval/PrecisionAtN.java | 21 ++++++++++++++++++- .../index/rankeval/PrecisionAtNTests.java | 10 +++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index c9ad3deb759..145270c15b1 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Objects; import javax.naming.directory.SearchResult; @@ -165,10 +166,28 @@ public class PrecisionAtN extends RankedListQualityMetric { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(NAME); + //builder.startObject(NAME); TODO Why does roundtripping fail with the name? + builder.startObject(); builder.field(SIZE_FIELD.getPreferredName(), this.n); builder.endObject(); return builder; } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PrecisionAtN other = (PrecisionAtN) obj; + return Objects.equals(n, other.n); + } + + @Override + public final int hashCode() { + return Objects.hash(getClass(), n); + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index d668b21630e..32b5c1cabc3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; -import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; @@ -38,7 +37,7 @@ import java.util.concurrent.ExecutionException; import static java.util.Collections.emptyList; -public class PrecisionAtNTests extends ESTestCase { +public class PrecisionAtNTests extends XContentRoundtripTestCase { public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); @@ -111,4 +110,11 @@ public class PrecisionAtNTests extends ESTestCase { partialResults.add(new EvalQueryQuality(0.6, emptyList())); assertEquals(0.3, metric.combine(partialResults), Double.MIN_VALUE); } + + public void testXContentRoundtrip() throws IOException { + int position = randomIntBetween(0, 1000); + + PrecisionAtN testItem = new PrecisionAtN(position); + roundtrip(testItem); + } } From cebb0ba0d8eb3003041c1082363ae10165b8726b Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 24 Aug 2016 14:41:58 +0200 Subject: [PATCH 034/297] Add roundtripping to ReciprocalRank --- .../index/rankeval/PrecisionAtN.java | 3 +-- .../index/rankeval/ReciprocalRank.java | 22 +++++++++++++++++-- .../index/rankeval/ReciprocalRankTests.java | 12 ++++++++-- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 145270c15b1..e9fa325a03f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -172,7 +172,7 @@ public class PrecisionAtN extends RankedListQualityMetric { builder.endObject(); return builder; } - + @Override public final boolean equals(Object obj) { if (this == obj) { @@ -189,5 +189,4 @@ public class PrecisionAtN extends RankedListQualityMetric { public final int hashCode() { return Objects.hash(getClass(), n); } - } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 791e857eca8..7b0576cd098 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import javax.naming.directory.SearchResult; @@ -159,10 +160,27 @@ public class ReciprocalRank extends RankedListQualityMetric { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.startObject(NAME); + //builder.startObject(NAME); builder.field(MAX_RANK_FIELD.getPreferredName(), this.maxAcceptableRank); - builder.endObject(); + //builder.endObject(); builder.endObject(); return builder; } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ReciprocalRank other = (ReciprocalRank) obj; + return Objects.equals(maxAcceptableRank, other.maxAcceptableRank); + } + + @Override + public final int hashCode() { + return Objects.hash(getClass(), maxAcceptableRank); + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index b524e763dc7..b25faba52d4 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -24,8 +24,8 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; -import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -33,7 +33,7 @@ import java.util.Vector; import static java.util.Collections.emptyList; -public class ReciprocalRankTests extends ESTestCase { +public class ReciprocalRankTests extends XContentRoundtripTestCase { public void testMaxAcceptableRank() { ReciprocalRank reciprocalRank = new ReciprocalRank(); @@ -123,4 +123,12 @@ public class ReciprocalRankTests extends ESTestCase { EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); } + + public void testXContentRoundtrip() throws IOException { + int position = randomIntBetween(0, 1000); + + ReciprocalRank testItem = new ReciprocalRank(position); + roundtrip(testItem); + } + } From 5979802415424d159902a23c6e72ef39726b9e3b Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 24 Aug 2016 15:17:42 +0200 Subject: [PATCH 035/297] Add comment wrt to changed xcontent generation --- .../java/org/elasticsearch/index/rankeval/ReciprocalRank.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 7b0576cd098..a40ecc1012e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -160,7 +160,7 @@ public class ReciprocalRank extends RankedListQualityMetric { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - //builder.startObject(NAME); + //builder.startObject(NAME); // TODO for roundtripping to work builder.field(MAX_RANK_FIELD.getPreferredName(), this.maxAcceptableRank); //builder.endObject(); builder.endObject(); From 94497871b5465b552b6bf91632887efdd51dc5bb Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 24 Aug 2016 15:17:58 +0200 Subject: [PATCH 036/297] Add roundtrip testing to QuerySpec --- .../index/rankeval/QuerySpec.java | 22 +++++++ .../index/rankeval/QuerySpecTests.java | 59 ++++++++++++++++++- .../index/rankeval/RatedDocumentTests.java | 9 ++- 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java index 92445bd4697..329839750b1 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java @@ -33,6 +33,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * Defines a QA specification: All end user supplied query intents will be mapped to the search request specified in this search request @@ -205,4 +206,25 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { builder.endObject(); return builder; } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + QuerySpec other = (QuerySpec) obj; + return Objects.equals(specId, other.specId) && + Objects.equals(testRequest, other.testRequest) && + Objects.equals(indices, other.indices) && + Objects.equals(types, other.types) && + Objects.equals(ratedDocs, other.ratedDocs); + } + + @Override + public final int hashCode() { + return Objects.hash(getClass(), specId, testRequest, indices.hashCode(), types.hashCode(), ratedDocs.hashCode()); + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java index 8d77e002a12..deec3f73b1a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -22,19 +22,26 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ParseFieldRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.test.ESTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import static java.util.Collections.emptyList; @@ -63,7 +70,55 @@ public class QuerySpecTests extends ESTestCase { searchRequestParsers = null; } - // TODO add some sort of roundtrip testing like we have now for queries? + public void testXContentRoundtrip() throws IOException { + String specId = randomAsciiOfLength(50); + + SearchSourceBuilder testRequest = new SearchSourceBuilder(); + testRequest.size(23); + testRequest.query(new MatchAllQueryBuilder()); + + List indices = new ArrayList<>(); + int size = randomIntBetween(0, 20); + for (int i = 0; i < size; i++) { + indices.add(randomAsciiOfLengthBetween(0, 50)); + } + + List types = new ArrayList<>(); + size = randomIntBetween(0, 20); + for (int i = 0; i < size; i++) { + types.add(randomAsciiOfLengthBetween(0, 50)); + } + + List ratedDocs = new ArrayList<>(); + size = randomIntBetween(0, 20); + for (int i = 0; i < size; i++) { + ratedDocs.add(RatedDocumentTests.createTestItem()); + } + + + QuerySpec testItem = new QuerySpec(specId, testRequest, indices, types, ratedDocs); + + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); + XContentBuilder shuffled = shuffleXContent(builder); + XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); + itemParser.nextToken(); // TODO this could be the reason why the metric roundtrip tests failed + + QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); + RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, + searchRequestParsers); + + QuerySpec parsedItem = QuerySpec.fromXContent(itemParser, rankContext); + parsedItem.setIndices(indices); // IRL these come from URL parameters - see RestRankEvalAction + parsedItem.setTypes(types); // IRL these come from URL parameters - see RestRankEvalAction + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + } + public void testParseFromXContent() throws IOException { String querySpecString = " {\n" + " \"id\": \"my_qa_query\",\n" @@ -99,4 +154,6 @@ public class QuerySpecTests extends ESTestCase { assertEquals("3", ratedDocs.get(2).getKey().getDocID()); assertEquals(1, ratedDocs.get(2).getRating()); } + + } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index 8cb4a194026..5188093defb 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -23,13 +23,16 @@ import java.io.IOException; public class RatedDocumentTests extends XContentRoundtripTestCase { - public void testXContentParsing() throws IOException { + public static RatedDocument createTestItem() { String index = randomAsciiOfLength(10); String type = randomAsciiOfLength(10); String docId = randomAsciiOfLength(10); int rating = randomInt(); - RatedDocument testItem = new RatedDocument(new RatedDocumentKey(index, type, docId), rating); - roundtrip(testItem); + return new RatedDocument(new RatedDocumentKey(index, type, docId), rating); + } + + public void testXContentParsing() throws IOException { + roundtrip(createTestItem()); } } From 87367be4e787aca6796392b2d5575bcd23980fd0 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 24 Aug 2016 15:32:44 +0200 Subject: [PATCH 037/297] Add roundtrip testing to RatedDocumentKey --- .../index/rankeval/RatedDocumentKey.java | 17 ++++++++-- .../index/rankeval/RatedDocumentKeyTests.java | 34 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java index c10bc3f0ce0..3efcc7e16a4 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -21,18 +21,20 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.FromXContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.Objects; -public class RatedDocumentKey extends ToXContentToBytes implements Writeable { +public class RatedDocumentKey extends ToXContentToBytes implements Writeable, FromXContentBuilder { public static final ParseField DOC_ID_FIELD = new ParseField("doc_id"); public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField INDEX_FIELD = new ParseField("index"); @@ -103,7 +105,18 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { out.writeString(type); out.writeString(docId); } - + + @Override + public RatedDocumentKey fromXContent(XContentParser parser, ParseFieldMatcher matcher) throws IOException { + return RatedDocumentKey.fromXContent(parser, new ParseFieldMatcherSupplier() { + + @Override + public ParseFieldMatcher getParseFieldMatcher() { + return matcher; + } + }); + } + public static RatedDocumentKey fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { return PARSER.apply(parser, context); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java new file mode 100644 index 00000000000..1ec1bbbcc06 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import java.io.IOException; + +public class RatedDocumentKeyTests extends XContentRoundtripTestCase { + + public void testXContentRoundtrip() throws IOException { + String index = randomAsciiOfLengthBetween(0, 10); + String type = randomAsciiOfLengthBetween(0, 10); + String docId = randomAsciiOfLengthBetween(0, 10); + + RatedDocumentKey testItem = new RatedDocumentKey(index, type, docId); + roundtrip(testItem); + } +} From f18e23eb51b89de16a2901f393a504835ad70984 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Thu, 25 Aug 2016 12:09:55 +0200 Subject: [PATCH 038/297] Fix test errors, add roundtrip testing to RankEvalSpec This adds roundtrip testing to RankEvalSpec, fixes issues introduced with the previous roundtrip tests, splits xcontent generation/parsing from actually checking the resulting objects to deal with e.g. all evaluation metrics needing some extra treatment. Renames QuerySpec to RatedRequest, renames newly introduced xcontent generation helper to conform with naming conventions. Fixes several lines that were too long, adds missing types where needed. --- .../rankeval/DiscountedCumulativeGainAt.java | 9 +- .../index/rankeval/PrecisionAtN.java | 3 +- .../index/rankeval/RankEvalSpec.java | 67 ++++++---- .../rankeval/RankedListQualityMetric.java | 6 +- .../index/rankeval/RatedDocument.java | 3 +- .../index/rankeval/RatedDocumentKey.java | 6 +- .../{QuerySpec.java => RatedRequest.java} | 22 ++-- .../index/rankeval/ReciprocalRank.java | 4 +- .../index/rankeval/RestRankEvalAction.java | 2 +- .../rankeval/TransportRankEvalAction.java | 8 +- .../DiscountedCumulativeGainAtTests.java | 17 ++- .../index/rankeval/PrecisionAtNTests.java | 17 ++- .../index/rankeval/QuerySpecTests.java | 25 ++-- .../index/rankeval/RankEvalRequestTests.java | 12 +- .../index/rankeval/RankEvalSpecTests.java | 119 ++++++++++++++++++ .../index/rankeval/RatedDocumentKeyTests.java | 11 +- .../index/rankeval/RatedDocumentTests.java | 12 +- .../index/rankeval/ReciprocalRankTests.java | 12 +- ...tCase.java => XContentRoundtripTests.java} | 11 +- 19 files changed, 275 insertions(+), 91 deletions(-) rename modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/{QuerySpec.java => RatedRequest.java} (90%) create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{XContentRoundtripTestCase.java => XContentRoundtripTests.java} (77%) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index 18e79bec3b9..22d98663738 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -86,8 +86,10 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - //builder.startObject(NAME); TODO Why does roundtripping fail with the name? builder.startObject(); + builder.startObject(NAME); builder.field(SIZE_FIELD.getPreferredName(), this.n); builder.endObject(); + builder.endObject(); return builder; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 7f88104b4eb..0aece1d0aa1 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Objects; /** * This class defines a ranking evaluation task including an id, a collection of queries to evaluate and the evaluation metric. @@ -43,9 +44,9 @@ import java.util.Collection; public class RankEvalSpec extends ToXContentToBytes implements Writeable { /** Collection of query specifications, that is e.g. search request templates to use for query translation. */ - private Collection specifications = new ArrayList<>(); + private Collection ratedRequests = new ArrayList<>(); /** Definition of the quality metric, e.g. precision at N */ - private RankedListQualityMetric eval; + private RankedListQualityMetric metric; /** a unique id for the whole QA task */ private String specId; @@ -53,34 +54,34 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { // TODO think if no args ctor is okay } - public RankEvalSpec(String specId, Collection specs, RankedListQualityMetric metric) { + public RankEvalSpec(String specId, Collection specs, RankedListQualityMetric metric) { this.specId = specId; - this.specifications = specs; - this.eval = metric; + this.ratedRequests = specs; + this.metric = metric; } public RankEvalSpec(StreamInput in) throws IOException { int specSize = in.readInt(); - specifications = new ArrayList<>(specSize); + ratedRequests = new ArrayList<>(specSize); for (int i = 0; i < specSize; i++) { - specifications.add(new QuerySpec(in)); + ratedRequests.add(new RatedRequest(in)); } - eval = in.readNamedWriteable(RankedListQualityMetric.class); + metric = in.readNamedWriteable(RankedListQualityMetric.class); specId = in.readString(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeInt(specifications.size()); - for (QuerySpec spec : specifications) { + out.writeInt(ratedRequests.size()); + for (RatedRequest spec : ratedRequests) { spec.writeTo(out); } - out.writeNamedWriteable(eval); + out.writeNamedWriteable(metric); out.writeString(specId); } - public void setEval(RankedListQualityMetric eval) { - this.eval = eval; + public void setEval(RankedListQualityMetric eval) { + this.metric = eval; } public void setTaskId(String taskId) { @@ -92,23 +93,23 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } /** Returns the precision at n configuration (containing level of n to consider).*/ - public RankedListQualityMetric getEvaluator() { - return eval; + public RankedListQualityMetric getEvaluator() { + return metric; } /** Sets the precision at n configuration (containing level of n to consider).*/ - public void setEvaluator(RankedListQualityMetric config) { - this.eval = config; + public void setEvaluator(RankedListQualityMetric config) { + this.metric = config; } /** Returns a list of intent to query translation specifications to evaluate. */ - public Collection getSpecifications() { - return specifications; + public Collection getSpecifications() { + return ratedRequests; } /** Set the list of intent to query translation specifications to evaluate. */ - public void setSpecifications(Collection specifications) { - this.specifications = specifications; + public void setSpecifications(Collection specifications) { + this.ratedRequests = specifications; } private static final ParseField SPECID_FIELD = new ParseField("spec_id"); @@ -127,7 +128,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } , METRIC_FIELD); PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> { try { - return QuerySpec.fromXContent(p, c); + return RatedRequest.fromXContent(p, c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } @@ -139,11 +140,11 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { builder.startObject(); builder.field(SPECID_FIELD.getPreferredName(), this.specId); builder.startArray(REQUESTS_FIELD.getPreferredName()); - for (QuerySpec spec : this.specifications) { + for (RatedRequest spec : this.ratedRequests) { spec.toXContent(builder, params); } builder.endArray(); - builder.field(METRIC_FIELD.getPreferredName(), this.eval); + builder.field(METRIC_FIELD.getPreferredName(), this.metric); builder.endObject(); return builder; } @@ -152,4 +153,22 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { return PARSER.parse(parser, context); } + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + RankEvalSpec other = (RankEvalSpec) obj; + return Objects.equals(specId, other.specId) && + Objects.equals(ratedRequests, other.ratedRequests) && + Objects.equals(metric, other.metric); + } + + @Override + public final int hashCode() { + return Objects.hash(getClass(), specId, ratedRequests, metric); + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index 30793d564e3..b5ce6121ffb 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -39,7 +39,7 @@ import java.util.List; * * RelevancyLevel specifies the type of object determining the relevancy level of some known docid. * */ -public abstract class RankedListQualityMetric +public abstract class RankedListQualityMetric> extends ToXContentToBytes implements NamedWriteable, FromXContentBuilder { @@ -52,8 +52,8 @@ public abstract class RankedListQualityMetric * */ public abstract EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs); - public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { - RankedListQualityMetric rc; + public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { + RankedListQualityMetric rc; Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name"); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index c5faecb986e..9b2dbf3fc09 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -43,7 +43,8 @@ public class RatedDocument extends ToXContentToBytes implements Writeable, FromX public static final ParseField RATING_FIELD = new ParseField("rating"); public static final ParseField KEY_FIELD = new ParseField("key"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rated_document", + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("rated_document", a -> new RatedDocument((RatedDocumentKey) a[0], (Integer) a[1])); static { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java index 3efcc7e16a4..cdffcf2ac57 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -39,7 +39,8 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable, Fr public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField INDEX_FIELD = new ParseField("index"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("ratings", + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("ratings", a -> new RatedDocumentKey((String) a[0], (String) a[1], (String) a[2])); static { @@ -117,7 +118,8 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable, Fr }); } - public static RatedDocumentKey fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { + public static RatedDocumentKey fromXContent( + XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { return PARSER.apply(parser, context); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java similarity index 90% rename from modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java rename to modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 329839750b1..2eb88a7f0e8 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -41,7 +41,7 @@ import java.util.Objects; * * The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery. * */ -public class QuerySpec extends ToXContentToBytes implements Writeable { +public class RatedRequest extends ToXContentToBytes implements Writeable { private String specId; private SearchSourceBuilder testRequest; @@ -50,12 +50,12 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { /** Collection of rated queries for this query QA specification.*/ private List ratedDocs = new ArrayList<>(); - public QuerySpec() { + public RatedRequest() { // ctor that doesn't require all args to be present immediatly is easier to use with ObjectParser // TODO decide if we can require only id as mandatory, set default values for the rest? } - public QuerySpec(String specId, SearchSourceBuilder testRequest, List indices, List types, + public RatedRequest(String specId, SearchSourceBuilder testRequest, List indices, List types, List ratedDocs) { this.specId = specId; this.testRequest = testRequest; @@ -64,7 +64,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { this.ratedDocs = ratedDocs; } - public QuerySpec(StreamInput in) throws IOException { + public RatedRequest(StreamInput in) throws IOException { this.specId = in.readString(); testRequest = new SearchSourceBuilder(in); int indicesSize = in.readInt(); @@ -149,18 +149,18 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { private static final ParseField ID_FIELD = new ParseField("id"); private static final ParseField REQUEST_FIELD = new ParseField("request"); private static final ParseField RATINGS_FIELD = new ParseField("ratings"); - private static final ObjectParser PARSER = new ObjectParser<>("requests", QuerySpec::new); + private static final ObjectParser PARSER = new ObjectParser<>("requests", RatedRequest::new); static { - PARSER.declareString(QuerySpec::setSpecId, ID_FIELD); - PARSER.declareObject(QuerySpec::setTestRequest, (p, c) -> { + PARSER.declareString(RatedRequest::setSpecId, ID_FIELD); + PARSER.declareObject(RatedRequest::setTestRequest, (p, c) -> { try { return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters()); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing request", ex); } } , REQUEST_FIELD); - PARSER.declareObjectArray(QuerySpec::setRatedDocs, (p, c) -> { + PARSER.declareObjectArray(RatedRequest::setRatedDocs, (p, c) -> { try { return RatedDocument.fromXContent(p, c); } catch (IOException ex) { @@ -170,7 +170,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { } /** - * Parses {@link QuerySpec} from rest representation: + * Parses {@link RatedRequest} from rest representation: * * Example: * { @@ -189,7 +189,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { * "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ] * } */ - public static QuerySpec fromXContent(XContentParser parser, RankEvalContext context) throws IOException { + public static RatedRequest fromXContent(XContentParser parser, RankEvalContext context) throws IOException { return PARSER.parse(parser, context); } @@ -215,7 +215,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { if (obj == null || getClass() != obj.getClass()) { return false; } - QuerySpec other = (QuerySpec) obj; + RatedRequest other = (RatedRequest) obj; return Objects.equals(specId, other.specId) && Objects.equals(testRequest, other.testRequest) && Objects.equals(indices, other.indices) && diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index a40ecc1012e..5e9d4874e4d 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -160,9 +160,9 @@ public class ReciprocalRank extends RankedListQualityMetric { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - //builder.startObject(NAME); // TODO for roundtripping to work + builder.startObject(NAME); builder.field(MAX_RANK_FIELD.getPreferredName(), this.maxAcceptableRank); - //builder.endObject(); + builder.endObject(); builder.endObject(); return builder; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 8acc7b34a51..e3e5941c969 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -196,7 +196,7 @@ public class RestRankEvalAction extends BaseRestHandler { List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); List types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type"))); RankEvalSpec spec = RankEvalSpec.parse(context.parser(), context); - for (QuerySpec specification : spec.getSpecifications()) { + for (RatedRequest specification : spec.getSpecifications()) { specification.setIndices(indices); specification.setTypes(types); }; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 3e4a8d0ea19..79332a22dc7 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -64,11 +64,11 @@ public class TransportRankEvalAction extends HandledTransportAction> unknownDocs = new ConcurrentHashMap<>(); - Collection specifications = qualityTask.getSpecifications(); + Collection specifications = qualityTask.getSpecifications(); AtomicInteger responseCounter = new AtomicInteger(specifications.size()); Map partialResults = new ConcurrentHashMap<>(specifications.size()); - for (QuerySpec querySpecification : specifications) { + for (RatedRequest querySpecification : specifications) { final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask, querySpecification, partialResults, unknownDocs, responseCounter); SearchSourceBuilder specRequest = querySpecification.getTestRequest(); @@ -85,13 +85,13 @@ public class TransportRankEvalAction extends HandledTransportAction { private ActionListener listener; - private QuerySpec specification; + private RatedRequest specification; private Map partialResults; private RankEvalSpec task; private Map> unknownDocs; private AtomicInteger responseCounter; - public RankEvalActionListener(ActionListener listener, RankEvalSpec task, QuerySpec specification, + public RankEvalActionListener(ActionListener listener, RankEvalSpec task, RatedRequest specification, Map partialResults, Map> unknownDocs, AtomicInteger responseCounter) { this.listener = listener; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java index 68207470382..e92f147acea 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java @@ -33,7 +33,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; -public class DiscountedCumulativeGainAtTests extends XContentRoundtripTestCase { +public class DiscountedCumulativeGainAtTests extends XContentRoundtripTests { /** * Assuming the docs are ranked in the following order: @@ -121,12 +121,21 @@ public class DiscountedCumulativeGainAtTests extends XContentRoundtripTestCase { +public class PrecisionAtNTests extends XContentRoundtripTests { public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); @@ -111,10 +111,19 @@ public class PrecisionAtNTests extends XContentRoundtripTestCase { assertEquals(0.3, metric.combine(partialResults), Double.MIN_VALUE); } - public void testXContentRoundtrip() throws IOException { + public static PrecisionAtN createTestItem() { int position = randomIntBetween(0, 1000); + return new PrecisionAtN(position); + } - PrecisionAtN testItem = new PrecisionAtN(position); - roundtrip(testItem); + public void testXContentRoundtrip() throws IOException { + PrecisionAtN testItem = createTestItem(); + XContentParser itemParser = roundtrip(testItem); + itemParser.nextToken(); + itemParser.nextToken(); + PrecisionAtN parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java index deec3f73b1a..887abf1a652 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -70,13 +70,23 @@ public class QuerySpecTests extends ESTestCase { searchRequestParsers = null; } - public void testXContentRoundtrip() throws IOException { + public static RatedRequest createTestItem(List indices, List types) { String specId = randomAsciiOfLength(50); SearchSourceBuilder testRequest = new SearchSourceBuilder(); testRequest.size(23); testRequest.query(new MatchAllQueryBuilder()); + List ratedDocs = new ArrayList<>(); + int size = randomIntBetween(0, 2); + for (int i = 0; i < size; i++) { + ratedDocs.add(RatedDocumentTests.createTestItem()); + } + + return new RatedRequest(specId, testRequest, indices, types, ratedDocs); + } + + public void testXContentRoundtrip() throws IOException { List indices = new ArrayList<>(); int size = randomIntBetween(0, 20); for (int i = 0; i < size; i++) { @@ -89,15 +99,8 @@ public class QuerySpecTests extends ESTestCase { types.add(randomAsciiOfLengthBetween(0, 50)); } - List ratedDocs = new ArrayList<>(); - size = randomIntBetween(0, 20); - for (int i = 0; i < size; i++) { - ratedDocs.add(RatedDocumentTests.createTestItem()); - } - + RatedRequest testItem = createTestItem(indices, types); - QuerySpec testItem = new QuerySpec(specId, testRequest, indices, types, ratedDocs); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); @@ -111,7 +114,7 @@ public class QuerySpecTests extends ESTestCase { RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, searchRequestParsers); - QuerySpec parsedItem = QuerySpec.fromXContent(itemParser, rankContext); + RatedRequest parsedItem = RatedRequest.fromXContent(itemParser, rankContext); parsedItem.setIndices(indices); // IRL these come from URL parameters - see RestRankEvalAction parsedItem.setTypes(types); // IRL these come from URL parameters - see RestRankEvalAction assertNotSame(testItem, parsedItem); @@ -142,7 +145,7 @@ public class QuerySpecTests extends ESTestCase { QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, searchRequestParsers); - QuerySpec specification = QuerySpec.fromXContent(parser, rankContext); + RatedRequest specification = RatedRequest.fromXContent(parser, rankContext); assertEquals("my_qa_query", specification.getSpecId()); assertNotNull(specification.getTestRequest()); List ratedDocs = specification.getRatedDocs(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index c03010c73b1..bae486ef0e3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -72,11 +72,11 @@ public class RankEvalRequestTests extends ESIntegTestCase { List types = Arrays.asList(new String[] { "testtype" }); String specId = randomAsciiOfLength(10); - List specifications = new ArrayList<>(); + List specifications = new ArrayList<>(); SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - specifications.add(new QuerySpec("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5"))); - specifications.add(new QuerySpec("berlin_query", testQuery, indices, types, createRelevant("1"))); + specifications.add(new RatedRequest("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5"))); + specifications.add(new RatedRequest("berlin_query", testQuery, indices, types, createRelevant("1"))); RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10)); @@ -106,14 +106,14 @@ public class RankEvalRequestTests extends ESIntegTestCase { List types = Arrays.asList(new String[] { "testtype" }); String specId = randomAsciiOfLength(10); - List specifications = new ArrayList<>(); + List specifications = new ArrayList<>(); SearchSourceBuilder amsterdamQuery = new SearchSourceBuilder(); amsterdamQuery.query(new MatchAllQueryBuilder()); - specifications.add(new QuerySpec("amsterdam_query", amsterdamQuery, indices, types, createRelevant("2", "3", "4", "5"))); + specifications.add(new RatedRequest("amsterdam_query", amsterdamQuery, indices, types, createRelevant("2", "3", "4", "5"))); SearchSourceBuilder brokenQuery = new SearchSourceBuilder(); RangeQueryBuilder brokenRangeQuery = new RangeQueryBuilder("text").timeZone("CET"); brokenQuery.query(brokenRangeQuery); - specifications.add(new QuerySpec("broken_query", brokenQuery, indices, types, createRelevant("1"))); + specifications.add(new RatedRequest("broken_query", brokenQuery, indices, types, createRelevant("1"))); RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10)); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java new file mode 100644 index 00000000000..3fac5cb3868 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -0,0 +1,119 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ParseFieldRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.SearchRequestParsers; +import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.suggest.Suggesters; +import org.elasticsearch.test.ESTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Collections.emptyList; + +public class RankEvalSpecTests extends ESTestCase { + private static SearchModule searchModule; + private static SearchRequestParsers searchRequestParsers; + + /** + * setup for the whole base test class + */ + @BeforeClass + public static void init() throws IOException { + AggregatorParsers aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"), + new ParseFieldRegistry<>("aggregation_pipes")); + searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); + IndicesQueriesRegistry queriesRegistry = searchModule.getQueryParserRegistry(); + Suggesters suggesters = searchModule.getSuggesters(); + searchRequestParsers = new SearchRequestParsers(queriesRegistry, aggsParsers, suggesters); + } + + @AfterClass + public static void afterClass() throws Exception { + searchModule = null; + searchRequestParsers = null; + } + + public void testRoundtripping() throws IOException { + List indices = new ArrayList<>(); + int size = randomIntBetween(0, 20); + for (int i = 0; i < size; i++) { + indices.add(randomAsciiOfLengthBetween(0, 50)); + } + + List types = new ArrayList<>(); + size = randomIntBetween(0, 20); + for (int i = 0; i < size; i++) { + types.add(randomAsciiOfLengthBetween(0, 50)); + } + List specs = new ArrayList<>(); + size = randomIntBetween(1, 2); // TODO I guess requests with no query spec should be rejected... + for (int i = 0; i < size; i++) { + specs.add(QuerySpecTests.createTestItem(indices, types)); + } + + String specId = randomAsciiOfLengthBetween(1, 10); // TODO we should reject zero length ids ... + @SuppressWarnings("rawtypes") + RankedListQualityMetric metric; + if (randomBoolean()) { + metric = PrecisionAtNTests.createTestItem(); + } else { + metric = DiscountedCumulativeGainAtTests.createTestItem(); + } + + RankEvalSpec testItem = new RankEvalSpec(specId, specs, metric); + + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); + XContentBuilder shuffled = shuffleXContent(builder); + XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); + + QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); + RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, + searchRequestParsers); + + RankEvalSpec parsedItem = RankEvalSpec.parse(itemParser, rankContext); + // IRL these come from URL parameters - see RestRankEvalAction + parsedItem.getSpecifications().stream().forEach(e -> {e.setIndices(indices); e.setTypes(types);}); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + } + +} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java index 1ec1bbbcc06..7111a0064a4 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java @@ -19,9 +19,12 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; + import java.io.IOException; -public class RatedDocumentKeyTests extends XContentRoundtripTestCase { +public class RatedDocumentKeyTests extends XContentRoundtripTests { public void testXContentRoundtrip() throws IOException { String index = randomAsciiOfLengthBetween(0, 10); @@ -29,6 +32,10 @@ public class RatedDocumentKeyTests extends XContentRoundtripTestCase { +public class RatedDocumentTests extends XContentRoundtripTests { public static RatedDocument createTestItem() { String index = randomAsciiOfLength(10); @@ -33,6 +36,11 @@ public class RatedDocumentTests extends XContentRoundtripTestCase } public void testXContentParsing() throws IOException { - roundtrip(createTestItem()); + RatedDocument testItem = createTestItem(); + XContentParser itemParser = roundtrip(testItem); + RatedDocument parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index b25faba52d4..56443b85c11 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -19,7 +19,9 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; import org.elasticsearch.search.SearchShardTarget; @@ -33,7 +35,7 @@ import java.util.Vector; import static java.util.Collections.emptyList; -public class ReciprocalRankTests extends XContentRoundtripTestCase { +public class ReciprocalRankTests extends XContentRoundtripTests { public void testMaxAcceptableRank() { ReciprocalRank reciprocalRank = new ReciprocalRank(); @@ -128,7 +130,13 @@ public class ReciprocalRankTests extends XContentRoundtripTestCase> extends ESTestCase { +public class XContentRoundtripTests> extends ESTestCase { - public void roundtrip(T testItem) throws IOException { + public XContentParser roundtrip(T testItem) throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); @@ -42,10 +41,6 @@ public class XContentRoundtripTestCase Date: Fri, 2 Sep 2016 18:03:58 +0200 Subject: [PATCH 039/297] Adding additional json object level to PrecisionAtN rendering --- .../java/org/elasticsearch/index/rankeval/PrecisionAtN.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index ba571c576ef..5c49e0f16ea 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -153,9 +153,11 @@ public class PrecisionAtN extends RankedListQualityMetric { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.startObject(NAME); builder.field(SIZE_FIELD.getPreferredName(), this.n); builder.endObject(); + builder.endObject(); return builder; } From 0b92d524a7d906c6f1ec8aa2af30c751b4117919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 2 Sep 2016 18:40:59 +0200 Subject: [PATCH 040/297] Add threshold for document ratings for PrecisionAtN and ReciprocalRank PrecisionAtN and ReciprocalRank are binary evaluation metrics by default that only distiguish between relevant/irrelevant search results. So far we assumed that relevant documents are labaled with 1 (irrelevant docs with 0) in the evaluation request, but this is cumbersome if the ratings are provided on a larger integer scale and would need to get mapped to a 0/1 value. This change introduces a threshold parameter on the PrecisionAtN and ReciprocalRank metric than can be used to set the threshold from which on a document is considered "relevant". It defaults to 1, so in case of 0/1 ratings the threshold doesn't have to be set and only ratings with value 0 are considered to be irrelevant. --- .../index/rankeval/PrecisionAtN.java | 37 ++++++++++++++---- .../index/rankeval/ReciprocalRank.java | 38 ++++++++++++++----- .../index/rankeval/PrecisionAtNTests.java | 25 +++++++++++- .../index/rankeval/ReciprocalRankTests.java | 25 ++++++++++++ 4 files changed, 106 insertions(+), 19 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 5c49e0f16ea..9aa04185e35 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -37,16 +37,30 @@ import javax.naming.directory.SearchResult; /** * Evaluate Precision at N, N being the number of search results to consider for precision calculation. - * * Documents of unkonwn quality are ignored in the precision at n computation and returned by document id. + * By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the precision + * calculation. This value can be changes using the "relevant_rating_threshold" parameter. * */ public class PrecisionAtN extends RankedListQualityMetric { /** Number of results to check against a given set of relevant results. */ private int n; + /** ratings equal or above this value will be considered relevant. */ + private int relevantRatingThreshhold = 1; + public static final String NAME = "precisionatn"; + private static final ParseField SIZE_FIELD = new ParseField("size"); + private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "precision_at", a -> new PrecisionAtN((Integer) a[0])); + + static { + PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD); + PARSER.declareInt(PrecisionAtN::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); + } + public PrecisionAtN(StreamInput in) throws IOException { n = in.readInt(); } @@ -82,12 +96,19 @@ public class PrecisionAtN extends RankedListQualityMetric { return n; } - private static final ParseField SIZE_FIELD = new ParseField("size"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "precision_at", a -> new PrecisionAtN((Integer) a[0])); + /** + * Sets the rating threshold above which ratings are considered to be "relevant" for this metric. + * */ + public void setRelevantRatingThreshhold(int threshold) { + this.relevantRatingThreshhold = threshold; + } - static { - PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD); + /** + * Return the rating threshold above which ratings are considered to be "relevant" for this metric. + * Defaults to 1. + * */ + public int getRelevantRatingThreshold() { + return relevantRatingThreshhold ; } public static PrecisionAtN fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { @@ -103,9 +124,9 @@ public class PrecisionAtN extends RankedListQualityMetric { Collection relevantDocIds = new ArrayList<>(); Collection irrelevantDocIds = new ArrayList<>(); for (RatedDocument doc : ratedDocs) { - if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + if (doc.getRating() >= this.relevantRatingThreshhold) { relevantDocIds.add(doc.getKey()); - } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + } else { irrelevantDocIds.add(doc.getKey()); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index dd4e710859b..3279c40734c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -26,8 +26,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; -import org.elasticsearch.index.rankeval.PrecisionAtN.RatingMapping; import org.elasticsearch.search.SearchHit; import java.io.IOException; @@ -41,6 +39,8 @@ import javax.naming.directory.SearchResult; /** * Evaluate reciprocal rank. + * By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the reciprocal rank + * calculation. This value can be changes using the "relevant_rating_threshold" parameter. * */ public class ReciprocalRank extends RankedListQualityMetric { @@ -48,6 +48,9 @@ public class ReciprocalRank extends RankedListQualityMetric { public static final int DEFAULT_MAX_ACCEPTABLE_RANK = 10; private int maxAcceptableRank = DEFAULT_MAX_ACCEPTABLE_RANK; + /** ratings equal or above this value will be considered relevant. */ + private int relevantRatingThreshhold = 1; + /** * Initializes maxAcceptableRank with 10 */ @@ -90,6 +93,21 @@ public class ReciprocalRank extends RankedListQualityMetric { return this.maxAcceptableRank; } + /** + * Sets the rating threshold above which ratings are considered to be "relevant" for this metric. + * */ + public void setRelevantRatingThreshhold(int threshold) { + this.relevantRatingThreshhold = threshold; + } + + /** + * Return the rating threshold above which ratings are considered to be "relevant" for this metric. + * Defaults to 1. + * */ + public int getRelevantRatingThreshold() { + return relevantRatingThreshhold ; + } + /** * Compute ReciprocalRank based on provided relevant document IDs. * @return reciprocal Rank for above {@link SearchResult} list. @@ -99,9 +117,9 @@ public class ReciprocalRank extends RankedListQualityMetric { Set relevantDocIds = new HashSet<>(); Set irrelevantDocIds = new HashSet<>(); for (RatedDocument doc : ratedDocs) { - if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + if (doc.getRating() >= this.relevantRatingThreshhold) { relevantDocIds.add(doc.getKey()); - } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { + } else { irrelevantDocIds.add(doc.getKey()); } } @@ -110,16 +128,14 @@ public class ReciprocalRank extends RankedListQualityMetric { int firstRelevant = -1; boolean found = false; for (int i = 0; i < hits.length; i++) { - // TODO here we use index/type/id triple not for a rated document but an unrated document in the search hits. Maybe rename? - RatedDocumentKey id = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); - if (relevantDocIds.contains(id)) { + RatedDocumentKey key = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); + if (relevantDocIds.contains(key)) { if (found == false && i < maxAcceptableRank) { - firstRelevant = i + 1; // add one because rank is not - // 0-based + firstRelevant = i + 1; // add one because rank is not 0-based found = true; } } else { - unknownDocIds.add(id); + unknownDocIds.add(key); } } @@ -133,11 +149,13 @@ public class ReciprocalRank extends RankedListQualityMetric { } private static final ParseField MAX_RANK_FIELD = new ParseField("max_acceptable_rank"); + private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); private static final ObjectParser PARSER = new ObjectParser<>( "reciprocal_rank", () -> new ReciprocalRank()); static { PARSER.declareInt(ReciprocalRank::setMaxAcceptableRank, MAX_RANK_FIELD); + PARSER.declareInt(ReciprocalRank::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); } public static ReciprocalRank fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index d668b21630e..381ddc10023 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -64,6 +64,27 @@ public class PrecisionAtNTests extends ESTestCase { assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); } + /** + * test that the relevant rating threshold can be set to something larger than 1. + * e.g. we set it to 2 here and expect dics 0-2 to be not relevant, doc 3 and 4 to be relevant + */ + public void testPrecisionAtFiveRelevanceThreshold() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), 0)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "1"), 1)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), 2)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), 3)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), 4)); + InternalSearchHit[] hits = new InternalSearchHit[5]; + for (int i = 0; i < 5; i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); + } + PrecisionAtN precisionAtN = new PrecisionAtN(5); + precisionAtN.setRelevantRatingThreshhold(2); + assertEquals((double) 3 / 5, precisionAtN.evaluate(hits, rated).getQualityLevel(), 0.00001); + } + public void testPrecisionAtFiveCorrectIndex() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); rated.add(new RatedDocument(new RatedDocumentKey("test_other", "testtype", "0"), Rating.RELEVANT.ordinal())); @@ -96,11 +117,13 @@ public class PrecisionAtNTests extends ESTestCase { public void testParseFromXContent() throws IOException { String xContent = " {\n" - + " \"size\": 10\n" + + " \"size\": 10,\n" + + " \"relevant_rating_threshold\" : 2" + "}"; XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent); PrecisionAtN precicionAt = PrecisionAtN.fromXContent(parser, () -> ParseFieldMatcher.STRICT); assertEquals(10, precicionAt.getN()); + assertEquals(2, precicionAt.getRelevantRatingThreshold()); } public void testCombine() { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index b524e763dc7..44c90d1eca3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -26,10 +26,12 @@ import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Vector; +import java.util.concurrent.ExecutionException; import static java.util.Collections.emptyList; @@ -103,6 +105,29 @@ public class ReciprocalRankTests extends ESTestCase { assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE); } + /** + * test that the relevant rating threshold can be set to something larger than 1. + * e.g. we set it to 2 here and expect dics 0-2 to be not relevant, so first relevant doc has + * third ranking position, so RR should be 1/3 + */ + public void testPrecisionAtFiveRelevanceThreshold() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), 0)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "1"), 1)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), 2)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), 3)); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), 4)); + InternalSearchHit[] hits = new InternalSearchHit[5]; + for (int i = 0; i < 5; i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); + } + + ReciprocalRank reciprocalRank = new ReciprocalRank(); + reciprocalRank.setRelevantRatingThreshhold(2); + assertEquals((double) 1 / 3, reciprocalRank.evaluate(hits, rated).getQualityLevel(), 0.00001); + } + public void testCombine() { ReciprocalRank reciprocalRank = new ReciprocalRank(); Vector partialResults = new Vector<>(3); From b8652b12248265c2d23dad5fa847e30d7a8931cf Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 11:21:29 +0200 Subject: [PATCH 041/297] Test wouldn't compile w/o these annotations. --- .../index/rankeval/RankEvalRequestTests.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index bae486ef0e3..35bc96146f3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -35,7 +35,15 @@ import java.util.List; import java.util.Map.Entry; import java.util.Set; -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) +@ESIntegTestCase.ClusterScope( + scope = ESIntegTestCase.Scope.SUITE, + minNumDataNodes = 1, + numDataNodes = 2, + maxNumDataNodes = 3, + numClientNodes = 1, + transportClientRatio = 0.5, + supportsDedicatedMasters = true, + randomDynamicTemplates = true) public class RankEvalRequestTests extends ESIntegTestCase { @Override protected Collection> transportClientPlugins() { From 6bc646ac84bc6e5d881da6106ac515abdb67025d Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 11:21:50 +0200 Subject: [PATCH 042/297] Make constructor enforce non-optional position argument. --- .../rankeval/DiscountedCumulativeGainAt.java | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index 22d98663738..9836def6d3b 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -50,24 +50,6 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric matcher); } public static DiscountedCumulativeGainAt fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { From 333b7698719d4d3fc88c3321f0525db9b2730de0 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 11:48:50 +0200 Subject: [PATCH 043/297] Remove usage of FromXContentBuilder --- .../index/rankeval/DiscountedCumulativeGainAt.java | 8 +------- .../elasticsearch/index/rankeval/PrecisionAtN.java | 14 +------------- .../elasticsearch/index/rankeval/RankEvalSpec.java | 10 +++++----- .../index/rankeval/RankedListQualityMetric.java | 9 +++------ .../index/rankeval/ReciprocalRank.java | 14 +------------- .../rankeval/DiscountedCumulativeGainAtTests.java | 2 +- .../index/rankeval/PrecisionAtNTests.java | 2 +- .../index/rankeval/RankEvalSpecTests.java | 1 - .../index/rankeval/ReciprocalRankTests.java | 2 +- .../index/rankeval/XContentRoundtripTests.java | 3 +-- 10 files changed, 15 insertions(+), 50 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index 9836def6d3b..07a3ac1a467 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -38,7 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; -public class DiscountedCumulativeGainAt extends RankedListQualityMetric { +public class DiscountedCumulativeGainAt extends RankedListQualityMetric { /** rank position up to which to check results. */ private int position; @@ -192,11 +191,6 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric matcher); - } - public static DiscountedCumulativeGainAt fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 50c2d54fa34..7d64dc31c8c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -43,7 +42,7 @@ import javax.naming.directory.SearchResult; * By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the precision * calculation. This value can be changes using the "relevant_rating_threshold" parameter. * */ -public class PrecisionAtN extends RankedListQualityMetric { +public class PrecisionAtN extends RankedListQualityMetric { /** Number of results to check against a given set of relevant results. */ private int n; @@ -113,17 +112,6 @@ public class PrecisionAtN extends RankedListQualityMetric { return relevantRatingThreshhold ; } - @Override - public PrecisionAtN fromXContent(XContentParser parser, ParseFieldMatcher matcher) { - return PrecisionAtN.fromXContent(parser, new ParseFieldMatcherSupplier() { - - @Override - public ParseFieldMatcher getParseFieldMatcher() { - return matcher; - } - }); - } - public static PrecisionAtN fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 0aece1d0aa1..f329ee17f9e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -46,7 +46,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { /** Collection of query specifications, that is e.g. search request templates to use for query translation. */ private Collection ratedRequests = new ArrayList<>(); /** Definition of the quality metric, e.g. precision at N */ - private RankedListQualityMetric metric; + private RankedListQualityMetric metric; /** a unique id for the whole QA task */ private String specId; @@ -54,7 +54,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { // TODO think if no args ctor is okay } - public RankEvalSpec(String specId, Collection specs, RankedListQualityMetric metric) { + public RankEvalSpec(String specId, Collection specs, RankedListQualityMetric metric) { this.specId = specId; this.ratedRequests = specs; this.metric = metric; @@ -80,7 +80,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { out.writeString(specId); } - public void setEval(RankedListQualityMetric eval) { + public void setEval(RankedListQualityMetric eval) { this.metric = eval; } @@ -93,12 +93,12 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } /** Returns the precision at n configuration (containing level of n to consider).*/ - public RankedListQualityMetric getEvaluator() { + public RankedListQualityMetric getEvaluator() { return metric; } /** Sets the precision at n configuration (containing level of n to consider).*/ - public void setEvaluator(RankedListQualityMetric config) { + public void setEvaluator(RankedListQualityMetric config) { this.metric = config; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index b5ce6121ffb..e423ab3533c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.NamedWriteable; -import org.elasticsearch.common.xcontent.FromXContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; @@ -39,9 +38,7 @@ import java.util.List; * * RelevancyLevel specifies the type of object determining the relevancy level of some known docid. * */ -public abstract class RankedListQualityMetric> - extends ToXContentToBytes - implements NamedWriteable, FromXContentBuilder { +public abstract class RankedListQualityMetric extends ToXContentToBytes implements NamedWriteable { /** * Returns a single metric representing the ranking quality of a set of returned documents @@ -52,8 +49,8 @@ public abstract class RankedListQualityMetric ratedDocs); - public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { - RankedListQualityMetric rc; + public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { + RankedListQualityMetric rc; Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name"); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 5907dba7a44..6d6626f9c4e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -44,7 +43,7 @@ import javax.naming.directory.SearchResult; * By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the reciprocal rank * calculation. This value can be changes using the "relevant_rating_threshold" parameter. * */ -public class ReciprocalRank extends RankedListQualityMetric { +public class ReciprocalRank extends RankedListQualityMetric { public static final String NAME = "reciprocal_rank"; public static final int DEFAULT_MAX_ACCEPTABLE_RANK = 10; @@ -160,17 +159,6 @@ public class ReciprocalRank extends RankedListQualityMetric { PARSER.declareInt(ReciprocalRank::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); } - @Override - public ReciprocalRank fromXContent(XContentParser parser, ParseFieldMatcher matcher) { - return ReciprocalRank.fromXContent(parser, new ParseFieldMatcherSupplier() { - - @Override - public ParseFieldMatcher getParseFieldMatcher() { - return matcher; - } - }); - } - public static ReciprocalRank fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java index e92f147acea..9bd93d1a9ad 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java @@ -133,7 +133,7 @@ public class DiscountedCumulativeGainAtTests extends XContentRoundtripTests ParseFieldMatcher.STRICT); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index 21aa9f6f957..4522d698bc7 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -144,7 +144,7 @@ public class PrecisionAtNTests extends XContentRoundtripTests { XContentParser itemParser = roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); - PrecisionAtN parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT); + PrecisionAtN parsedItem = PrecisionAtN.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 3fac5cb3868..7bc2c7649a3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -86,7 +86,6 @@ public class RankEvalSpecTests extends ESTestCase { } String specId = randomAsciiOfLengthBetween(1, 10); // TODO we should reject zero length ids ... - @SuppressWarnings("rawtypes") RankedListQualityMetric metric; if (randomBoolean()) { metric = PrecisionAtNTests.createTestItem(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index f0d9cacdc38..db232662ea8 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -157,7 +157,7 @@ public class ReciprocalRankTests extends XContentRoundtripTests XContentParser itemParser = roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); - ReciprocalRank parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT); + ReciprocalRank parsedItem = ReciprocalRank.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTests.java index 14bd3d20f79..a37055d897a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; -import org.elasticsearch.common.xcontent.FromXContentBuilder; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -31,7 +30,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; -public class XContentRoundtripTests> extends ESTestCase { +public class XContentRoundtripTests extends ESTestCase { public XContentParser roundtrip(T testItem) throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); From efbef20361e4ca04fe8493ecb3a4a49441a524db Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 11:53:39 +0200 Subject: [PATCH 044/297] Remove call to getClass from hashCode implementations. --- .../index/rankeval/DiscountedCumulativeGainAt.java | 2 +- .../java/org/elasticsearch/index/rankeval/PrecisionAtN.java | 2 +- .../java/org/elasticsearch/index/rankeval/RankEvalSpec.java | 2 +- .../java/org/elasticsearch/index/rankeval/RatedDocument.java | 2 +- .../java/org/elasticsearch/index/rankeval/RatedDocumentKey.java | 2 +- .../java/org/elasticsearch/index/rankeval/RatedRequest.java | 2 +- .../java/org/elasticsearch/index/rankeval/ReciprocalRank.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index 07a3ac1a467..36d2a208353 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -225,6 +225,6 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { @Override public final int hashCode() { - return Objects.hash(getClass(), position, normalize, unknownDocRating); + return Objects.hash(position, normalize, unknownDocRating); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 7d64dc31c8c..ac8675a2a37 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -197,6 +197,6 @@ public class PrecisionAtN extends RankedListQualityMetric { @Override public final int hashCode() { - return Objects.hash(getClass(), n); + return Objects.hash(n); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index f329ee17f9e..39e937aaf6c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -169,6 +169,6 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { @Override public final int hashCode() { - return Objects.hash(getClass(), specId, ratedRequests, metric); + return Objects.hash(specId, ratedRequests, metric); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index 9b2dbf3fc09..8a9ad2071e2 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -136,6 +136,6 @@ public class RatedDocument extends ToXContentToBytes implements Writeable, FromX @Override public final int hashCode() { - return Objects.hash(getClass(), key, rating); + return Objects.hash(key, rating); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java index cdffcf2ac57..feef20b890c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -139,6 +139,6 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable, Fr @Override public final int hashCode() { - return Objects.hash(getClass(), index, type, docId); + return Objects.hash(index, type, docId); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 2eb88a7f0e8..86861a2c95e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -225,6 +225,6 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { @Override public final int hashCode() { - return Objects.hash(getClass(), specId, testRequest, indices.hashCode(), types.hashCode(), ratedDocs.hashCode()); + return Objects.hash(specId, testRequest, indices.hashCode(), types.hashCode(), ratedDocs.hashCode()); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 6d6626f9c4e..803900a3521 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -187,6 +187,6 @@ public class ReciprocalRank extends RankedListQualityMetric { @Override public final int hashCode() { - return Objects.hash(getClass(), maxAcceptableRank); + return Objects.hash(maxAcceptableRank); } } From c8a3b3c32fd15b8f972aa530951f1a0d6930a14e Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 12:02:00 +0200 Subject: [PATCH 045/297] Move xcontent roundtrip method to helper class. --- .../rankeval/DiscountedCumulativeGainAtTests.java | 5 +++-- .../index/rankeval/PrecisionAtNTests.java | 5 +++-- .../index/rankeval/RatedDocumentKeyTests.java | 5 +++-- .../index/rankeval/RatedDocumentTests.java | 5 +++-- .../index/rankeval/ReciprocalRankTests.java | 5 +++-- ...tentRoundtripTests.java => XContentTestHelper.java} | 10 +++++----- 6 files changed, 20 insertions(+), 15 deletions(-) rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{XContentRoundtripTests.java => XContentTestHelper.java} (84%) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java index 9bd93d1a9ad..1ef63e16405 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; @@ -33,7 +34,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; -public class DiscountedCumulativeGainAtTests extends XContentRoundtripTests { +public class DiscountedCumulativeGainAtTests extends ESTestCase { /** * Assuming the docs are ranked in the following order: @@ -130,7 +131,7 @@ public class DiscountedCumulativeGainAtTests extends XContentRoundtripTests ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index 4522d698bc7..14a4be5ec38 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; @@ -37,7 +38,7 @@ import java.util.concurrent.ExecutionException; import static java.util.Collections.emptyList; -public class PrecisionAtNTests extends XContentRoundtripTests { +public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); @@ -141,7 +142,7 @@ public class PrecisionAtNTests extends XContentRoundtripTests { public void testXContentRoundtrip() throws IOException { PrecisionAtN testItem = createTestItem(); - XContentParser itemParser = roundtrip(testItem); + XContentParser itemParser = XContentTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); PrecisionAtN parsedItem = PrecisionAtN.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java index 7111a0064a4..8ac79b67b2c 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java @@ -21,10 +21,11 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; import java.io.IOException; -public class RatedDocumentKeyTests extends XContentRoundtripTests { +public class RatedDocumentKeyTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { String index = randomAsciiOfLengthBetween(0, 10); @@ -32,7 +33,7 @@ public class RatedDocumentKeyTests extends XContentRoundtripTests { +public class RatedDocumentTests extends ESTestCase { public static RatedDocument createTestItem() { String index = randomAsciiOfLength(10); @@ -37,7 +38,7 @@ public class RatedDocumentTests extends XContentRoundtripTests { public void testXContentParsing() throws IOException { RatedDocument testItem = createTestItem(); - XContentParser itemParser = roundtrip(testItem); + XContentParser itemParser = XContentTestHelper.roundtrip(testItem); RatedDocument parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index db232662ea8..40fb7a05eea 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; @@ -36,7 +37,7 @@ import java.util.concurrent.ExecutionException; import static java.util.Collections.emptyList; -public class ReciprocalRankTests extends XContentRoundtripTests { +public class ReciprocalRankTests extends ESTestCase { public void testMaxAcceptableRank() { ReciprocalRank reciprocalRank = new ReciprocalRank(); @@ -154,7 +155,7 @@ public class ReciprocalRankTests extends XContentRoundtripTests int position = randomIntBetween(0, 1000); ReciprocalRank testItem = new ReciprocalRank(position); - XContentParser itemParser = roundtrip(testItem); + XContentParser itemParser = XContentTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); ReciprocalRank parsedItem = ReciprocalRank.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentTestHelper.java similarity index 84% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTests.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentTestHelper.java index a37055d897a..e02772ad6f6 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentTestHelper.java @@ -30,15 +30,15 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; -public class XContentRoundtripTests extends ESTestCase { +public class XContentTestHelper { - public XContentParser roundtrip(T testItem) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - if (randomBoolean()) { + public static XContentParser roundtrip(ToXContentToBytes testItem) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(ESTestCase.randomFrom(XContentType.values())); + if (ESTestCase.randomBoolean()) { builder.prettyPrint(); } testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); - XContentBuilder shuffled = shuffleXContent(builder); + XContentBuilder shuffled = ESTestCase.shuffleXContent(builder); XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); return itemParser; } From b3a4a89151175f23ff4f09202c28e35a87d0da42 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 12:06:39 +0200 Subject: [PATCH 046/297] Replace magic number in test with random number. --- .../java/org/elasticsearch/index/rankeval/QuerySpecTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java index 887abf1a652..86b421c7ac3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -74,7 +74,7 @@ public class QuerySpecTests extends ESTestCase { String specId = randomAsciiOfLength(50); SearchSourceBuilder testRequest = new SearchSourceBuilder(); - testRequest.size(23); + testRequest.size(randomInt()); testRequest.query(new MatchAllQueryBuilder()); List ratedDocs = new ArrayList<>(); @@ -108,7 +108,7 @@ public class QuerySpecTests extends ESTestCase { testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); XContentBuilder shuffled = shuffleXContent(builder); XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); - itemParser.nextToken(); // TODO this could be the reason why the metric roundtrip tests failed + itemParser.nextToken(); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, From 0d25fc4925d8b29296e774b9772d5d4f1c70f305 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 12:11:07 +0200 Subject: [PATCH 047/297] Re-use roundtrip helper for QuerySpecTests --- .../index/rankeval/QuerySpecTests.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java index 86b421c7ac3..48232854c9b 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -22,12 +22,8 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ParseFieldRegistry; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.indices.query.IndicesQueriesRegistry; @@ -100,14 +96,7 @@ public class QuerySpecTests extends ESTestCase { } RatedRequest testItem = createTestItem(indices, types); - - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - if (randomBoolean()) { - builder.prettyPrint(); - } - testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); - XContentBuilder shuffled = shuffleXContent(builder); - XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); + XContentParser itemParser = XContentTestHelper.roundtrip(testItem); itemParser.nextToken(); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); From 2af0218bdd467ceeba3af80909109f1619a765eb Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 12:12:06 +0200 Subject: [PATCH 048/297] Rename test class to match renamed implementation Used to be QuerySpec, is now RatedRequest; changing name of test accordingly. --- .../org/elasticsearch/index/rankeval/RankEvalSpecTests.java | 2 +- .../rankeval/{QuerySpecTests.java => RatedRequestsTests.java} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{QuerySpecTests.java => RatedRequestsTests.java} (99%) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 7bc2c7649a3..fc1cf0abafd 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -82,7 +82,7 @@ public class RankEvalSpecTests extends ESTestCase { List specs = new ArrayList<>(); size = randomIntBetween(1, 2); // TODO I guess requests with no query spec should be rejected... for (int i = 0; i < size; i++) { - specs.add(QuerySpecTests.createTestItem(indices, types)); + specs.add(RatedRequestsTests.createTestItem(indices, types)); } String specId = randomAsciiOfLengthBetween(1, 10); // TODO we should reject zero length ids ... diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java similarity index 99% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 48232854c9b..7104b6aa4f4 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -42,7 +42,7 @@ import java.util.List; import static java.util.Collections.emptyList; -public class QuerySpecTests extends ESTestCase { +public class RatedRequestsTests extends ESTestCase { private static SearchModule searchModule; private static SearchRequestParsers searchRequestParsers; From 9d8bb720dce1c455e983d5b5e1c469004c8babcf Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 12:50:04 +0200 Subject: [PATCH 049/297] Remove leftover FromXContentBuilder reference --- .../index/rankeval/RatedDocument.java | 15 +-------------- .../index/rankeval/RatedDocumentTests.java | 2 +- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index 8a9ad2071e2..1cde3828507 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -21,14 +21,12 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.FromXContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -38,7 +36,7 @@ import java.util.Objects; /** * A document ID and its rating for the query QA use case. * */ -public class RatedDocument extends ToXContentToBytes implements Writeable, FromXContentBuilder { +public class RatedDocument extends ToXContentToBytes implements Writeable { public static final ParseField RATING_FIELD = new ParseField("rating"); public static final ParseField KEY_FIELD = new ParseField("key"); @@ -97,17 +95,6 @@ public class RatedDocument extends ToXContentToBytes implements Writeable, FromX out.writeVInt(rating); } - @Override - public RatedDocument fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException { - return RatedDocument.fromXContent(parser, new ParseFieldMatcherSupplier() { - - @Override - public ParseFieldMatcher getParseFieldMatcher() { - return parseFieldMatcher; - } - }); - } - public static RatedDocument fromXContent(XContentParser parser, ParseFieldMatcherSupplier supplier) throws IOException { return PARSER.apply(parser, supplier); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index 37dfb28b160..3084682ad75 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -39,7 +39,7 @@ public class RatedDocumentTests extends ESTestCase { public void testXContentParsing() throws IOException { RatedDocument testItem = createTestItem(); XContentParser itemParser = XContentTestHelper.roundtrip(testItem); - RatedDocument parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT); + RatedDocument parsedItem = RatedDocument.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); From ba9956a46894f9770f4121d44a41fefa8b5b7abd Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 12:52:30 +0200 Subject: [PATCH 050/297] Remove FromXContentBuilder from RatedDocumentKey --- .../index/rankeval/RatedDocumentKey.java | 15 +-------------- .../index/rankeval/RatedDocumentKeyTests.java | 2 +- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java index feef20b890c..5e69f902dea 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -21,20 +21,18 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.FromXContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.Objects; -public class RatedDocumentKey extends ToXContentToBytes implements Writeable, FromXContentBuilder { +public class RatedDocumentKey extends ToXContentToBytes implements Writeable { public static final ParseField DOC_ID_FIELD = new ParseField("doc_id"); public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField INDEX_FIELD = new ParseField("index"); @@ -107,17 +105,6 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable, Fr out.writeString(docId); } - @Override - public RatedDocumentKey fromXContent(XContentParser parser, ParseFieldMatcher matcher) throws IOException { - return RatedDocumentKey.fromXContent(parser, new ParseFieldMatcherSupplier() { - - @Override - public ParseFieldMatcher getParseFieldMatcher() { - return matcher; - } - }); - } - public static RatedDocumentKey fromXContent( XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { return PARSER.apply(parser, context); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java index 8ac79b67b2c..6c0bd766b35 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java @@ -34,7 +34,7 @@ public class RatedDocumentKeyTests extends ESTestCase { RatedDocumentKey testItem = new RatedDocumentKey(index, type, docId); XContentParser itemParser = XContentTestHelper.roundtrip(testItem); - RatedDocumentKey parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT); + RatedDocumentKey parsedItem = RatedDocumentKey.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); From d34a117422f76057c0cf5dd1f7e825b8a71f7a02 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 13:03:18 +0200 Subject: [PATCH 051/297] Use roundtrip helper for rank eval spec tests. --- .../index/rankeval/RankEvalSpecTests.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index fc1cf0abafd..b6a6025a0ef 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -22,12 +22,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ParseFieldRegistry; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.SearchModule; @@ -95,13 +90,7 @@ public class RankEvalSpecTests extends ESTestCase { RankEvalSpec testItem = new RankEvalSpec(specId, specs, metric); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - if (randomBoolean()) { - builder.prettyPrint(); - } - testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); - XContentBuilder shuffled = shuffleXContent(builder); - XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); + XContentParser itemParser = XContentTestHelper.roundtrip(testItem); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, From 450b756152ba212737647a6d7291ac9f1c00536a Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Sep 2016 13:54:27 +0200 Subject: [PATCH 052/297] We don't actually need this annotation anymore. --- .../index/rankeval/RankEvalRequestTests.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index 35bc96146f3..3a61b3c98ec 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -35,15 +35,6 @@ import java.util.List; import java.util.Map.Entry; import java.util.Set; -@ESIntegTestCase.ClusterScope( - scope = ESIntegTestCase.Scope.SUITE, - minNumDataNodes = 1, - numDataNodes = 2, - maxNumDataNodes = 3, - numClientNodes = 1, - transportClientRatio = 0.5, - supportsDedicatedMasters = true, - randomDynamicTemplates = true) public class RankEvalRequestTests extends ESIntegTestCase { @Override protected Collection> transportClientPlugins() { From 63822c2745182177c8f33d38e0bf175017bd25d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 12 Sep 2016 12:05:17 +0200 Subject: [PATCH 053/297] Adapt to changes on master --- .../org/elasticsearch/index/rankeval/RankEvalContext.java | 5 +++++ .../org/elasticsearch/index/rankeval/RatedRequest.java | 6 +++--- .../elasticsearch/index/rankeval/RankEvalSpecTests.java | 4 ++-- .../elasticsearch/index/rankeval/RatedRequestsTests.java | 8 ++++---- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java index 496dcbca02a..576fc594c9a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.search.SearchExtRegistry; import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.suggest.Suggesters; @@ -47,6 +48,10 @@ public class RankEvalContext implements ParseFieldMatcherSupplier { return searchRequestParsers.aggParsers; } + public SearchExtRegistry getSearchExtParsers() { + return searchRequestParsers.searchExtParsers; + } + @Override public ParseFieldMatcher getParseFieldMatcher() { return this.parseFieldMatcher; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 86861a2c95e..f40f004bad7 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -155,7 +155,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { PARSER.declareString(RatedRequest::setSpecId, ID_FIELD); PARSER.declareObject(RatedRequest::setTestRequest, (p, c) -> { try { - return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters()); + return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters(), c.getSearchExtParsers()); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing request", ex); } @@ -206,7 +206,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { builder.endObject(); return builder; } - + @Override public final boolean equals(Object obj) { if (this == obj) { @@ -222,7 +222,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { Objects.equals(types, other.types) && Objects.equals(ratedDocs, other.ratedDocs); } - + @Override public final int hashCode() { return Objects.hash(specId, testRequest, indices.hashCode(), types.hashCode(), ratedDocs.hashCode()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index b6a6025a0ef..b7296161b05 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -53,7 +53,7 @@ public class RankEvalSpecTests extends ESTestCase { searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); IndicesQueriesRegistry queriesRegistry = searchModule.getQueryParserRegistry(); Suggesters suggesters = searchModule.getSuggesters(); - searchRequestParsers = new SearchRequestParsers(queriesRegistry, aggsParsers, suggesters); + searchRequestParsers = new SearchRequestParsers(queriesRegistry, aggsParsers, suggesters, null); } @AfterClass @@ -98,7 +98,7 @@ public class RankEvalSpecTests extends ESTestCase { RankEvalSpec parsedItem = RankEvalSpec.parse(itemParser, rankContext); // IRL these come from URL parameters - see RestRankEvalAction - parsedItem.getSpecifications().stream().forEach(e -> {e.setIndices(indices); e.setTypes(types);}); + parsedItem.getSpecifications().stream().forEach(e -> {e.setIndices(indices); e.setTypes(types);}); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 7104b6aa4f4..d8f901cfb33 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -57,7 +57,7 @@ public class RatedRequestsTests extends ESTestCase { searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); IndicesQueriesRegistry queriesRegistry = searchModule.getQueryParserRegistry(); Suggesters suggesters = searchModule.getSuggesters(); - searchRequestParsers = new SearchRequestParsers(queriesRegistry, aggsParsers, suggesters); + searchRequestParsers = new SearchRequestParsers(queriesRegistry, aggsParsers, suggesters, null); } @AfterClass @@ -78,7 +78,7 @@ public class RatedRequestsTests extends ESTestCase { for (int i = 0; i < size; i++) { ratedDocs.add(RatedDocumentTests.createTestItem()); } - + return new RatedRequest(specId, testRequest, indices, types, ratedDocs); } @@ -146,6 +146,6 @@ public class RatedRequestsTests extends ESTestCase { assertEquals("3", ratedDocs.get(2).getKey().getDocID()); assertEquals(1, ratedDocs.get(2).getRating()); } - - + + } From 7dc1c3349d392eaae06ac9420a1e28579ca0de13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 12 Sep 2016 15:43:32 +0200 Subject: [PATCH 054/297] Remove nested `key` field for rated documents Every rated document needs an index/type/id parameter, so adding a "key" object like we currently do only leads to an additional unneeded level of nesting in the rest request. Closes #20417 --- .../index/rankeval/RankEvalResponse.java | 17 +++-- .../index/rankeval/RatedDocument.java | 65 ++++++++++--------- .../index/rankeval/RatedDocumentKey.java | 41 +----------- .../index/rankeval/RestRankEvalAction.java | 40 +++++------- .../DiscountedCumulativeGainAtTests.java | 6 +- .../index/rankeval/PrecisionAtNTests.java | 42 ++++++------ .../index/rankeval/RankEvalRequestTests.java | 2 +- .../index/rankeval/RankEvalResponseTests.java | 14 ++++ .../index/rankeval/RatedDocumentKeyTests.java | 41 +++++++++--- .../index/rankeval/RatedDocumentTests.java | 2 +- .../index/rankeval/RatedRequestsTests.java | 25 +++---- .../index/rankeval/ReciprocalRankTests.java | 26 +++----- .../test/rank_eval/10_basic.yaml | 20 +++--- .../rest-api-spec/test/rank_eval/20_dcg.yaml | 48 +++++++------- 14 files changed, 197 insertions(+), 192 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 9d58d47847f..48e1406b1f6 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -117,13 +117,20 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { builder.startObject("rank_eval"); builder.field("spec_id", specId); builder.field("quality_level", qualityLevel); - builder.startArray("unknown_docs"); + builder.startObject("unknown_docs"); for (String key : unknownDocs.keySet()) { - builder.startObject(); - builder.field(key, unknownDocs.get(key)); - builder.endObject(); + Collection keys = unknownDocs.get(key); + builder.startArray(key); + for (RatedDocumentKey docKey : keys) { + builder.startObject(); + builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), docKey.getIndex()); + builder.field(RatedDocument.TYPE_FIELD.getPreferredName(), docKey.getType()); + builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), docKey.getDocID()); + builder.endObject(); + } + builder.endArray(); } - builder.endArray(); + builder.endObject(); builder.endObject(); return builder; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index 1cde3828507..d8af409445c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -22,7 +22,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcherSupplier; -import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -39,41 +38,26 @@ import java.util.Objects; public class RatedDocument extends ToXContentToBytes implements Writeable { public static final ParseField RATING_FIELD = new ParseField("rating"); - public static final ParseField KEY_FIELD = new ParseField("key"); + public static final ParseField DOC_ID_FIELD = new ParseField("doc_id"); + public static final ParseField TYPE_FIELD = new ParseField("type"); + public static final ParseField INDEX_FIELD = new ParseField("index"); private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("rated_document", - a -> new RatedDocument((RatedDocumentKey) a[0], (Integer) a[1])); - + new ConstructingObjectParser<>("rated_document", + a -> new RatedDocument((String) a[0], (String) a[1], (String) a[2], (Integer) a[3])); + static { - PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { - try { - return RatedDocumentKey.fromXContent(p, c); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); - } - } , KEY_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), DOC_ID_FIELD); PARSER.declareInt(ConstructingObjectParser.constructorArg(), RATING_FIELD); } - private RatedDocumentKey key; private int rating; + private RatedDocumentKey key; - void setRatedDocumentKey(RatedDocumentKey key) { - this.key = key; - } - - void setKey(RatedDocumentKey key) { - this.key = key; - } - - void setRating(int rating) { - this.rating = rating; - } - - public RatedDocument(RatedDocumentKey key, int rating) { - this.key = key; - this.rating = rating; + public RatedDocument(String index, String type, String docId, int rating) { + this(new RatedDocumentKey(index, type, docId), rating); } public RatedDocument(StreamInput in) throws IOException { @@ -81,10 +65,27 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { this.rating = in.readVInt(); } + public RatedDocument(RatedDocumentKey ratedDocumentKey, int rating) { + this.key = ratedDocumentKey; + this.rating = rating; + } + public RatedDocumentKey getKey() { return this.key; } + public String getIndex() { + return key.getIndex(); + } + + public String getType() { + return key.getType(); + } + + public String getDocID() { + return key.getDocID(); + } + public int getRating() { return rating; } @@ -98,11 +99,13 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { public static RatedDocument fromXContent(XContentParser parser, ParseFieldMatcherSupplier supplier) throws IOException { return PARSER.apply(parser, supplier); } - + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(KEY_FIELD.getPreferredName(), key); + builder.field(INDEX_FIELD.getPreferredName(), key.getIndex()); + builder.field(TYPE_FIELD.getPreferredName(), key.getType()); + builder.field(DOC_ID_FIELD.getPreferredName(), key.getDocID()); builder.field(RATING_FIELD.getPreferredName(), rating); builder.endObject(); return builder; @@ -120,7 +123,7 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { return Objects.equals(key, other.key) && Objects.equals(rating, other.rating); } - + @Override public final int hashCode() { return Objects.hash(key, rating); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java index 5e69f902dea..ee08cf018da 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -19,45 +19,15 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.action.support.ToXContentToBytes; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.Objects; -public class RatedDocumentKey extends ToXContentToBytes implements Writeable { - public static final ParseField DOC_ID_FIELD = new ParseField("doc_id"); - public static final ParseField TYPE_FIELD = new ParseField("type"); - public static final ParseField INDEX_FIELD = new ParseField("index"); +public class RatedDocumentKey implements Writeable { - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("ratings", - a -> new RatedDocumentKey((String) a[0], (String) a[1], (String) a[2])); - - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_FIELD); - PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); - PARSER.declareString(ConstructingObjectParser.constructorArg(), DOC_ID_FIELD); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(INDEX_FIELD.getPreferredName(), index); - builder.field(TYPE_FIELD.getPreferredName(), type); - builder.field(DOC_ID_FIELD.getPreferredName(), docId); - builder.endObject(); - return builder; - } - - // TODO instead of docId use path to id and id itself private String docId; private String type; private String index; @@ -69,7 +39,7 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { void setType(String type) { this.type = type; } - + void setDocId(String docId) { this.docId = docId; } @@ -105,11 +75,6 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { out.writeString(docId); } - public static RatedDocumentKey fromXContent( - XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { - return PARSER.apply(parser, context); - } - @Override public final boolean equals(Object obj) { if (this == obj) { @@ -123,7 +88,7 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { Objects.equals(type, other.type) && Objects.equals(docId, other.docId); } - + @Override public final int hashCode() { return Objects.hash(index, type, docId); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index e3e5941c969..aebede0ff50 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -69,7 +69,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; "requests": [{ "id": "amsterdam_query", "request": { - "query": { + "query": { "bool": { "must": [ {"match": {"beverage": "coffee"}}, @@ -78,15 +78,12 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; {"term": {"ip_location": {"value": "ams","boost": 10}}}]} }, "size": 10 - } }, - "ratings": { - "1": 1, - "2": 0, - "3": 1, - "4": 1 - } - } + "ratings": [ + {\"index\": \"test\", \"type\": \"my_type\", \"doc_id\": \"1\", \"rating\" : 1 }, + {\"index\": \"test\", \"type\": \"my_type\", \"doc_id\": \"2\", \"rating\" : 0 }, + {\"index\": \"test\", \"type\": \"my_type\", \"doc_id\": \"3\", \"rating\" : 1 } + ] }, { "id": "berlin_query", "request": { @@ -99,13 +96,8 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; {"term": {"ip_location": {"value": "ber","boost": 10}}}]} }, "size": 10 - } }, - "ratings": { - "1": 0, - "5": 1, - "6": 1 - } + "ratings": [ ... ] }], "metric": { "precisionAtN": { @@ -129,7 +121,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; "failed": 0 }, "quality_level": ... quality level ..., - "unknown_docs": [{"user_request_id": [... list of unknown docs ...]}] + "unknown_docs": {"user_request_id": [... list of unknown docs ...]} } * @@ -148,11 +140,17 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; "rank_eval": [{ "spec_id": "huge_weight_on_location", "quality_level": 0.4, - "unknown_docs": [{ - "amsterdam_query": [5, 10, 23] + "unknown_docs": { + "amsterdam_query": [ + { "index" : "test", "type" : "my_type", "doc_id" : "21"}, + { "index" : "test", "type" : "my_type", "doc_id" : "5"}, + { "index" : "test", "type" : "my_type", "doc_id" : "9"} + ] }, { - "berlin_query": [42] - }] + "berlin_query": [ + { "index" : "test", "type" : "my_type", "doc_id" : "42"} + ] + } }] } @@ -189,8 +187,6 @@ public class RestRankEvalAction extends BaseRestHandler { client.execute(RankEvalAction.INSTANCE, rankEvalRequest, new RestToXContentListener(channel)); } - - public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context) throws IOException { List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java index 1ef63e16405..8ae55e314ac 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java @@ -55,7 +55,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { int[] relevanceRatings = new int[] { 3, 2, 3, 0, 1, 2 }; InternalSearchHit[] hits = new InternalSearchHit[6]; for (int i = 0; i < 6; i++) { - rated.add(new RatedDocument(new RatedDocumentKey("index", "type", Integer.toString(i)), relevanceRatings[i])); + rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } @@ -100,7 +100,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { InternalSearchHit[] hits = new InternalSearchHit[6]; for (int i = 0; i < 6; i++) { if (i < relevanceRatings.length) { - rated.add(new RatedDocument(new RatedDocumentKey("index", "type", Integer.toString(i)), relevanceRatings[i])); + rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); } hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); @@ -121,7 +121,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { assertEquals(8, dcgAt.getPosition()); assertEquals(true, dcgAt.getNormalize()); } - + public static DiscountedCumulativeGainAt createTestItem() { int position = randomIntBetween(0, 1000); boolean normalize = randomBoolean(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index 14a4be5ec38..bc78484832f 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -42,7 +42,7 @@ public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); InternalSearchHit[] hits = new InternalSearchHit[1]; hits[0] = new InternalSearchHit(0, "0", new Text("testtype"), Collections.emptyMap()); hits[0].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); @@ -51,11 +51,11 @@ public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "1"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), Rating.IRRELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); InternalSearchHit[] hits = new InternalSearchHit[5]; for (int i = 0; i < 5; i++) { hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); @@ -70,11 +70,11 @@ public class PrecisionAtNTests extends ESTestCase { */ public void testPrecisionAtFiveRelevanceThreshold() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), 0)); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "1"), 1)); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), 2)); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), 3)); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), 4)); + rated.add(new RatedDocument("test", "testtype", "0", 0)); + rated.add(new RatedDocument("test", "testtype", "1", 1)); + rated.add(new RatedDocument("test", "testtype", "2", 2)); + rated.add(new RatedDocument("test", "testtype", "3", 3)); + rated.add(new RatedDocument("test", "testtype", "4", 4)); InternalSearchHit[] hits = new InternalSearchHit[5]; for (int i = 0; i < 5; i++) { hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); @@ -87,11 +87,11 @@ public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveCorrectIndex() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument(new RatedDocumentKey("test_other", "testtype", "0"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test_other", "testtype", "1"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), Rating.IRRELEVANT.ordinal())); + rated.add(new RatedDocument("test_other", "testtype", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test_other", "testtype", "1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); InternalSearchHit[] hits = new InternalSearchHit[5]; for (int i = 0; i < 5; i++) { hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); @@ -102,11 +102,11 @@ public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveCorrectType() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument(new RatedDocumentKey("test", "other_type", "0"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test", "other_type", "1"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), Rating.IRRELEVANT.ordinal())); + rated.add(new RatedDocument("test", "other_type", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "other_type", "1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); InternalSearchHit[] hits = new InternalSearchHit[5]; for (int i = 0; i < 5; i++) { hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index 3a61b3c98ec..bb49debf6fe 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -125,7 +125,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { private static List createRelevant(String... docs) { List relevant = new ArrayList<>(); for (String doc : docs) { - relevant.add(new RatedDocument(new RatedDocumentKey("test", "testtype", doc), Rating.RELEVANT.ordinal())); + relevant.add(new RatedDocument("test", "testtype", doc, Rating.RELEVANT.ordinal())); } return relevant; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index 89e63b0b6f0..67119e2789f 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -21,6 +21,10 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -61,4 +65,14 @@ public class RankEvalResponseTests extends ESTestCase { } } + public void testToXContent() throws IOException { + RankEvalResponse randomResponse = createRandomResponse(); + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + if (ESTestCase.randomBoolean()) { + builder.prettyPrint(); + } + builder.startObject(); + randomResponse.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java index 6c0bd766b35..26661c086a6 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java @@ -19,24 +19,49 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + public class RatedDocumentKeyTests extends ESTestCase { - public void testXContentRoundtrip() throws IOException { + public void testEqualsAndHash() throws IOException { String index = randomAsciiOfLengthBetween(0, 10); String type = randomAsciiOfLengthBetween(0, 10); String docId = randomAsciiOfLengthBetween(0, 10); RatedDocumentKey testItem = new RatedDocumentKey(index, type, docId); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); - RatedDocumentKey parsedItem = RatedDocumentKey.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); - assertNotSame(testItem, parsedItem); - assertEquals(testItem, parsedItem); - assertEquals(testItem.hashCode(), parsedItem.hashCode()); + + assertFalse("key is equal to null", testItem.equals(null)); + assertFalse("key is equal to incompatible type", testItem.equals("")); + assertTrue("key is not equal to self", testItem.equals(testItem)); + assertThat("same key's hashcode returns different values if called multiple times", testItem.hashCode(), + equalTo(testItem.hashCode())); + + RatedDocumentKey mutation; + switch (randomIntBetween(0, 2)) { + case 0: + mutation = new RatedDocumentKey(testItem.getIndex() + "_foo", testItem.getType(), testItem.getDocID()); + break; + case 1: + mutation = new RatedDocumentKey(testItem.getIndex(), testItem.getType() + "_foo", testItem.getDocID()); + break; + case 2: + mutation = new RatedDocumentKey(testItem.getIndex(), testItem.getType(), testItem.getDocID() + "_foo"); + break; + default: + throw new IllegalStateException("The test should only allow three parameters mutated"); + } + + assertThat("different keys should not be equal", mutation, not(equalTo(testItem))); + + RatedDocumentKey secondEqualKey = new RatedDocumentKey(index, type, docId); + assertTrue("key is not equal to its copy", testItem.equals(secondEqualKey)); + assertTrue("equals is not symmetric", secondEqualKey.equals(testItem)); + assertThat("key copy's hashcode is different from original hashcode", secondEqualKey.hashCode(), + equalTo(testItem.hashCode())); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index 3084682ad75..b3af34423c7 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -33,7 +33,7 @@ public class RatedDocumentTests extends ESTestCase { String docId = randomAsciiOfLength(10); int rating = randomInt(); - return new RatedDocument(new RatedDocumentKey(index, type, docId), rating); + return new RatedDocument(index, type, docId, rating); } public void testXContentParsing() throws IOException { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index d8f901cfb33..dd7a7d02ac0 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -112,6 +112,7 @@ public class RatedRequestsTests extends ESTestCase { } public void testParseFromXContent() throws IOException { + // we modify the order of index/type/docId to make sure it doesn't matter for parsing xContent String querySpecString = " {\n" + " \"id\": \"my_qa_query\",\n" + " \"request\": {\n" @@ -126,9 +127,9 @@ public class RatedRequestsTests extends ESTestCase { + " \"size\": 10\n" + " },\n" + " \"ratings\": [ " - + " {\"key\": {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"1\"}, \"rating\" : 1 }, " - + " {\"key\": {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"2\"}, \"rating\" : 0 }, " - + " {\"key\": {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"3\"}, \"rating\" : 1 }]\n" + + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"1\", \"rating\" : 1 }, " + + " {\"type\": \"testtype\", \"index\": \"test\", \"doc_id\": \"2\", \"rating\" : 0 }, " + + " {\"doc_id\": \"3\", \"index\": \"test\", \"type\": \"testtype\", \"rating\" : 1 }]\n" + "}"; XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, ParseFieldMatcher.STRICT); @@ -139,13 +140,15 @@ public class RatedRequestsTests extends ESTestCase { assertNotNull(specification.getTestRequest()); List ratedDocs = specification.getRatedDocs(); assertEquals(3, ratedDocs.size()); - assertEquals("1", ratedDocs.get(0).getKey().getDocID()); - assertEquals(1, ratedDocs.get(0).getRating()); - assertEquals("2", ratedDocs.get(1).getKey().getDocID()); - assertEquals(0, ratedDocs.get(1).getRating()); - assertEquals("3", ratedDocs.get(2).getKey().getDocID()); - assertEquals(1, ratedDocs.get(2).getRating()); + for (int i = 0; i < 3; i++) { + assertEquals("" + (i + 1), ratedDocs.get(i).getDocID()); + assertEquals("test", ratedDocs.get(i).getIndex()); + assertEquals("testtype", ratedDocs.get(i).getType()); + if (i == 1) { + assertEquals(0, ratedDocs.get(i).getRating()); + } else { + assertEquals(1, ratedDocs.get(i).getRating()); + } + } } - - } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 40fb7a05eea..c3f951b09b4 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -56,13 +56,9 @@ public class ReciprocalRankTests extends ESTestCase { int relevantAt = 5; for (int i = 0; i < 10; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument( - new RatedDocumentKey("test", "type", Integer.toString(i)), - Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), Rating.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument( - new RatedDocumentKey("test", "type", Integer.toString(i)), - Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), Rating.IRRELEVANT.ordinal())); } } @@ -93,13 +89,9 @@ public class ReciprocalRankTests extends ESTestCase { int relevantAt = randomIntBetween(0, 9); for (int i = 0; i <= 20; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument( - new RatedDocumentKey("test", "type", Integer.toString(i)), - Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), Rating.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument( - new RatedDocumentKey("test", "type", Integer.toString(i)), - Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), Rating.IRRELEVANT.ordinal())); } } @@ -114,11 +106,11 @@ public class ReciprocalRankTests extends ESTestCase { */ public void testPrecisionAtFiveRelevanceThreshold() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), 0)); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "1"), 1)); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), 2)); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), 3)); - rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), 4)); + rated.add(new RatedDocument("test", "testtype", "0", 0)); + rated.add(new RatedDocument("test", "testtype", "1", 1)); + rated.add(new RatedDocument("test", "testtype", "2", 2)); + rated.add(new RatedDocument("test", "testtype", "3", 3)); + rated.add(new RatedDocument("test", "testtype", "4", 4)); InternalSearchHit[] hits = new InternalSearchHit[5]; for (int i = 0; i < 5; i++) { hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 7dd41141b33..234dc827ebf 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -47,14 +47,14 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, "ratings": [ - {"key": { "index": "foo", "type": "bar", "doc_id": "doc1"}, "rating": 0}, - {"key": { "index": "foo", "type": "bar", "doc_id": "doc2"}, "rating": 1}, - {"key": { "index": "foo", "type": "bar", "doc_id": "doc3"}, "rating": 1}] + {"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 0}, + {"index": "foo", "type": "bar", "doc_id": "doc2", "rating": 1}, + {"index": "foo", "type": "bar", "doc_id": "doc3", "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, - "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc1"}, "rating": 1}] + "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 1}] } ], "metric" : { "precisionatn": { "size": 10}} @@ -62,8 +62,8 @@ - match: {rank_eval.spec_id: "cities_qa_queries"} - match: {rank_eval.quality_level: 1} - - match: {rank_eval.unknown_docs.0.amsterdam_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} - - match: {rank_eval.unknown_docs.1.berlin_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} + - match: {rank_eval.unknown_docs.amsterdam_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} + - match: {rank_eval.unknown_docs.berlin_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} --- "Reciprocal Rank": @@ -114,13 +114,13 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, # doc4 should be returned in third position, so reciprocal rank is 1/3 - "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] + "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 - "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] + "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 1}] } ], "metric" : { "reciprocal_rank": {} } @@ -139,13 +139,13 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, # doc4 should be returned in third position, so reciprocal rank is 1/3 - "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] + "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 - "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] + "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 1}] } ], "metric" : { diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml index f122265e5a7..3bd4e42b62c 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml @@ -53,12 +53,12 @@ "id": "dcg_query", "request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] }, "ratings": [ - {"key": {"index": "foo", "type": "bar", "doc_id": "doc1"}, "rating": 3}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc2"}, "rating": 2}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc3"}, "rating": 3}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 0}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc5"}, "rating": 1}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc6"}, "rating": 2}] + {"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 3}, + {"index": "foo", "type": "bar", "doc_id": "doc2", "rating": 2}, + {"index": "foo", "type": "bar", "doc_id": "doc3", "rating": 3}, + {"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 0}, + {"index": "foo", "type": "bar", "doc_id": "doc5", "rating": 1}, + {"index": "foo", "type": "bar", "doc_id": "doc6", "rating": 2}] } ], "metric" : { "dcg_at_n": { "size": 6}} @@ -78,12 +78,12 @@ "id": "dcg_query_reverse", "request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] }, "ratings": [ - {"key": {"index": "foo", "type": "bar", "doc_id": "doc1"}, "rating": 3}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc2"}, "rating": 2}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc3"}, "rating": 3}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 0}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc5"}, "rating": 1}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc6"}, "rating": 2}] + {"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 3}, + {"index": "foo", "type": "bar", "doc_id": "doc2", "rating": 2}, + {"index": "foo", "type": "bar", "doc_id": "doc3", "rating": 3}, + {"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 0}, + {"index": "foo", "type": "bar", "doc_id": "doc5", "rating": 1}, + {"index": "foo", "type": "bar", "doc_id": "doc6", "rating": 2}] }, ], "metric" : { "dcg_at_n": { "size": 6}} @@ -103,23 +103,23 @@ "id": "dcg_query", "request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] }, "ratings": [ - {"key": {"index": "foo", "type": "bar", "doc_id": "doc1"}, "rating": 3}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc2"}, "rating": 2}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc3"}, "rating": 3}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 0}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc5"}, "rating": 1}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc6"}, "rating": 2}] + {"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 3}, + {"index": "foo", "type": "bar", "doc_id": "doc2", "rating": 2}, + {"index": "foo", "type": "bar", "doc_id": "doc3", "rating": 3}, + {"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 0}, + {"index": "foo", "type": "bar", "doc_id": "doc5", "rating": 1}, + {"index": "foo", "type": "bar", "doc_id": "doc6", "rating": 2}] }, { "id": "dcg_query_reverse", "request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] }, "ratings": [ - {"key": {"index": "foo", "type": "bar", "doc_id": "doc1"}, "rating": 3}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc2"}, "rating": 2}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc3"}, "rating": 3}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 0}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc5"}, "rating": 1}, - {"key": {"index": "foo", "type": "bar", "doc_id": "doc6"}, "rating": 2}] + {"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 3}, + {"index": "foo", "type": "bar", "doc_id": "doc2", "rating": 2}, + {"index": "foo", "type": "bar", "doc_id": "doc3", "rating": 3}, + {"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 0}, + {"index": "foo", "type": "bar", "doc_id": "doc5", "rating": 1}, + {"index": "foo", "type": "bar", "doc_id": "doc6", "rating": 2}] }, ], "metric" : { "dcg_at_n": { "size": 6}} From 5398425f4f79342ee3d7b780b6c39faaca848b2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 13 Sep 2016 11:09:10 +0200 Subject: [PATCH 055/297] Remove top level spec_id Currently the top level spec_id serves as a human-readable description of the ranking evaluation API call. Since there is only one id possible, it can be dropped to simplify the request. Closes #20438 --- .../index/rankeval/RankEvalResponse.java | 20 +++------------ .../index/rankeval/RankEvalSpec.java | 25 +++---------------- .../rankeval/TransportRankEvalAction.java | 2 +- .../index/rankeval/RankEvalRequestTests.java | 7 ++---- .../index/rankeval/RankEvalResponseTests.java | 2 +- .../index/rankeval/RankEvalSpecTests.java | 3 +-- .../test/rank_eval/10_basic.yaml | 10 ++------ .../rest-api-spec/test/rank_eval/20_dcg.yaml | 6 ----- 8 files changed, 15 insertions(+), 60 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 48e1406b1f6..43b66b46861 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -44,8 +44,6 @@ import java.util.Objects; **/ //TODO instead of just returning averages over complete results, think of other statistics, micro avg, macro avg, partial results public class RankEvalResponse extends ActionResponse implements ToXContent { - /**ID of QA specification this result was generated for.*/ - private String specId; /**Average precision observed when issuing query intents with this specification.*/ private double qualityLevel; /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ @@ -54,17 +52,11 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { public RankEvalResponse() { } - public RankEvalResponse(String specId, double qualityLevel, Map> unknownDocs) { - this.specId = specId; + public RankEvalResponse(double qualityLevel, Map> unknownDocs) { this.qualityLevel = qualityLevel; this.unknownDocs = unknownDocs; } - - public String getSpecId() { - return specId; - } - public double getQualityLevel() { return qualityLevel; } @@ -75,13 +67,12 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { @Override public String toString() { - return "RankEvalResponse, ID :[" + specId + "], quality: " + qualityLevel + ", unknown docs: " + unknownDocs; + return "RankEvalResponse, quality: " + qualityLevel + ", unknown docs: " + unknownDocs; } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeString(specId); out.writeDouble(qualityLevel); out.writeVInt(unknownDocs.size()); for (String queryId : unknownDocs.keySet()) { @@ -97,7 +88,6 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - this.specId = in.readString(); this.qualityLevel = in.readDouble(); int unknownDocumentSets = in.readVInt(); this.unknownDocs = new HashMap<>(unknownDocumentSets); @@ -115,7 +105,6 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("rank_eval"); - builder.field("spec_id", specId); builder.field("quality_level", qualityLevel); builder.startObject("unknown_docs"); for (String key : unknownDocs.keySet()) { @@ -144,13 +133,12 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { return false; } RankEvalResponse other = (RankEvalResponse) obj; - return Objects.equals(specId, other.specId) && - Objects.equals(qualityLevel, other.qualityLevel) && + return Objects.equals(qualityLevel, other.qualityLevel) && Objects.equals(unknownDocs, other.unknownDocs); } @Override public final int hashCode() { - return Objects.hash(getClass(), specId, qualityLevel, unknownDocs); + return Objects.hash(getClass(), qualityLevel, unknownDocs); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 39e937aaf6c..cadb88d77c0 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -47,15 +47,12 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { private Collection ratedRequests = new ArrayList<>(); /** Definition of the quality metric, e.g. precision at N */ private RankedListQualityMetric metric; - /** a unique id for the whole QA task */ - private String specId; public RankEvalSpec() { // TODO think if no args ctor is okay } - public RankEvalSpec(String specId, Collection specs, RankedListQualityMetric metric) { - this.specId = specId; + public RankEvalSpec(Collection specs, RankedListQualityMetric metric) { this.ratedRequests = specs; this.metric = metric; } @@ -67,7 +64,6 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { ratedRequests.add(new RatedRequest(in)); } metric = in.readNamedWriteable(RankedListQualityMetric.class); - specId = in.readString(); } @Override @@ -77,21 +73,12 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { spec.writeTo(out); } out.writeNamedWriteable(metric); - out.writeString(specId); } public void setEval(RankedListQualityMetric eval) { this.metric = eval; } - public void setTaskId(String taskId) { - this.specId = taskId; - } - - public String getTaskId() { - return this.specId; - } - /** Returns the precision at n configuration (containing level of n to consider).*/ public RankedListQualityMetric getEvaluator() { return metric; @@ -112,13 +99,11 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { this.ratedRequests = specifications; } - private static final ParseField SPECID_FIELD = new ParseField("spec_id"); private static final ParseField METRIC_FIELD = new ParseField("metric"); private static final ParseField REQUESTS_FIELD = new ParseField("requests"); private static final ObjectParser PARSER = new ObjectParser<>("rank_eval", RankEvalSpec::new); static { - PARSER.declareString(RankEvalSpec::setTaskId, SPECID_FIELD); PARSER.declareObject(RankEvalSpec::setEvaluator, (p, c) -> { try { return RankedListQualityMetric.fromXContent(p, c); @@ -138,7 +123,6 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(SPECID_FIELD.getPreferredName(), this.specId); builder.startArray(REQUESTS_FIELD.getPreferredName()); for (RatedRequest spec : this.ratedRequests) { spec.toXContent(builder, params); @@ -162,13 +146,12 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { return false; } RankEvalSpec other = (RankEvalSpec) obj; - return Objects.equals(specId, other.specId) && - Objects.equals(ratedRequests, other.ratedRequests) && + return Objects.equals(ratedRequests, other.ratedRequests) && Objects.equals(metric, other.metric); } - + @Override public final int hashCode() { - return Objects.hash(specId, ratedRequests, metric); + return Objects.hash(ratedRequests, metric); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 79332a22dc7..f7f21eb81e7 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -112,7 +112,7 @@ public class TransportRankEvalAction extends HandledTransportAction indices = Arrays.asList(new String[] { "test" }); List types = Arrays.asList(new String[] { "testtype" }); - String specId = randomAsciiOfLength(10); List specifications = new ArrayList<>(); SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); specifications.add(new RatedRequest("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5"))); specifications.add(new RatedRequest("berlin_query", testQuery, indices, types, createRelevant("1"))); - RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10)); + RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtN(10)); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); - assertEquals(specId, response.getSpecId()); assertEquals(1.0, response.getQualityLevel(), Double.MIN_VALUE); Set>> entrySet = response.getUnknownDocs().entrySet(); assertEquals(2, entrySet.size()); @@ -104,7 +102,6 @@ public class RankEvalRequestTests extends ESIntegTestCase { List indices = Arrays.asList(new String[] { "test" }); List types = Arrays.asList(new String[] { "testtype" }); - String specId = randomAsciiOfLength(10); List specifications = new ArrayList<>(); SearchSourceBuilder amsterdamQuery = new SearchSourceBuilder(); amsterdamQuery.query(new MatchAllQueryBuilder()); @@ -114,7 +111,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { brokenQuery.query(brokenRangeQuery); specifications.add(new RatedRequest("broken_query", brokenQuery, indices, types, createRelevant("1"))); - RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10)); + RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtN(10)); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index 67119e2789f..b5b68ec22ae 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -47,7 +47,7 @@ public class RankEvalResponseTests extends ESTestCase { } unknownDocs.put(randomAsciiOfLength(5), ids); } - return new RankEvalResponse(randomAsciiOfLengthBetween(1, 10), randomDouble(), unknownDocs ); + return new RankEvalResponse(randomDouble(), unknownDocs ); } public void testSerialization() throws IOException { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index b7296161b05..97fd0a9f281 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -80,7 +80,6 @@ public class RankEvalSpecTests extends ESTestCase { specs.add(RatedRequestsTests.createTestItem(indices, types)); } - String specId = randomAsciiOfLengthBetween(1, 10); // TODO we should reject zero length ids ... RankedListQualityMetric metric; if (randomBoolean()) { metric = PrecisionAtNTests.createTestItem(); @@ -88,7 +87,7 @@ public class RankEvalSpecTests extends ESTestCase { metric = DiscountedCumulativeGainAtTests.createTestItem(); } - RankEvalSpec testItem = new RankEvalSpec(specId, specs, metric); + RankEvalSpec testItem = new RankEvalSpec(specs, metric); XContentParser itemParser = XContentTestHelper.roundtrip(testItem); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 234dc827ebf..679f6014df3 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -40,8 +40,7 @@ - do: rank_eval: - body: { - "spec_id" : "cities_qa_queries", + body: { "requests" : [ { "id": "amsterdam_query", @@ -60,7 +59,6 @@ "metric" : { "precisionatn": { "size": 10}} } - - match: {rank_eval.spec_id: "cities_qa_queries"} - match: {rank_eval.quality_level: 1} - match: {rank_eval.unknown_docs.amsterdam_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} - match: {rank_eval.unknown_docs.berlin_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} @@ -107,8 +105,7 @@ - do: rank_eval: - body: { - "spec_id" : "cities_qa_queries", + body: { "requests" : [ { "id": "amsterdam_query", @@ -126,14 +123,12 @@ "metric" : { "reciprocal_rank": {} } } - - match: {rank_eval.spec_id: "cities_qa_queries"} # average is (1/3 + 1/2)/2 = 5/12 ~ 0.41666666666666663 - match: {rank_eval.quality_level: 0.41666666666666663} - do: rank_eval: body: { - "spec_id" : "cities_qa_queries", "requests" : [ { "id": "amsterdam_query", @@ -155,6 +150,5 @@ } } - - match: {rank_eval.spec_id: "cities_qa_queries"} # average is (0 + 1/2)/2 = 1/4 - match: {rank_eval.quality_level: 0.25} diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml index 3bd4e42b62c..5df0b45fcc9 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml @@ -47,7 +47,6 @@ - do: rank_eval: body: { - "spec_id" : "dcg_qa_queries", "requests" : [ { "id": "dcg_query", @@ -64,7 +63,6 @@ "metric" : { "dcg_at_n": { "size": 6}} } - - match: {rank_eval.spec_id: "dcg_qa_queries"} - match: {rank_eval.quality_level: 13.84826362927298} # reverse the order in which the results are returned (less relevant docs first) @@ -72,7 +70,6 @@ - do: rank_eval: body: { - "spec_id" : "dcg_qa_queries", "requests" : [ { "id": "dcg_query_reverse", @@ -89,7 +86,6 @@ "metric" : { "dcg_at_n": { "size": 6}} } - - match: {rank_eval.spec_id: "dcg_qa_queries"} - match: {rank_eval.quality_level: 10.29967439154499} # if we mix both, we should get the average @@ -97,7 +93,6 @@ - do: rank_eval: body: { - "spec_id" : "dcg_qa_queries", "requests" : [ { "id": "dcg_query", @@ -125,5 +120,4 @@ "metric" : { "dcg_at_n": { "size": 6}} } - - match: {rank_eval.spec_id: "dcg_qa_queries"} - match: {rank_eval.quality_level: 12.073969010408984} From 6c1003b81a414cd651bf7f5bcbc9bd13f0232448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 21 Sep 2016 12:36:50 +0200 Subject: [PATCH 056/297] Normalize rated document key parameter To be consitent with the output of the search API, we should use the same field names for specifying the document ("_index", "_type", "_id") when providing the rated documents in the `rank_eval` request. --- .../index/rankeval/RatedDocument.java | 6 +-- .../index/rankeval/RatedRequestsTests.java | 6 +-- .../test/rank_eval/10_basic.yaml | 20 ++++---- .../rest-api-spec/test/rank_eval/20_dcg.yaml | 48 +++++++++---------- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index d8af409445c..e0e608cc4ae 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -38,9 +38,9 @@ import java.util.Objects; public class RatedDocument extends ToXContentToBytes implements Writeable { public static final ParseField RATING_FIELD = new ParseField("rating"); - public static final ParseField DOC_ID_FIELD = new ParseField("doc_id"); - public static final ParseField TYPE_FIELD = new ParseField("type"); - public static final ParseField INDEX_FIELD = new ParseField("index"); + public static final ParseField DOC_ID_FIELD = new ParseField("_id"); + public static final ParseField TYPE_FIELD = new ParseField("_type"); + public static final ParseField INDEX_FIELD = new ParseField("_index"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rated_document", diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index dd7a7d02ac0..0d1615a5d05 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -127,9 +127,9 @@ public class RatedRequestsTests extends ESTestCase { + " \"size\": 10\n" + " },\n" + " \"ratings\": [ " - + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"1\", \"rating\" : 1 }, " - + " {\"type\": \"testtype\", \"index\": \"test\", \"doc_id\": \"2\", \"rating\" : 0 }, " - + " {\"doc_id\": \"3\", \"index\": \"test\", \"type\": \"testtype\", \"rating\" : 1 }]\n" + + " {\"_index\": \"test\", \"_type\": \"testtype\", \"_id\": \"1\", \"rating\" : 1 }, " + + " {\"_type\": \"testtype\", \"_index\": \"test\", \"_id\": \"2\", \"rating\" : 0 }, " + + " {\"_id\": \"3\", \"_index\": \"test\", \"_type\": \"testtype\", \"rating\" : 1 }]\n" + "}"; XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 679f6014df3..2abad7353e9 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -46,22 +46,22 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, "ratings": [ - {"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 0}, - {"index": "foo", "type": "bar", "doc_id": "doc2", "rating": 1}, - {"index": "foo", "type": "bar", "doc_id": "doc3", "rating": 1}] + {"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 0}, + {"_index": "foo", "_type": "bar", "_id": "doc2", "rating": 1}, + {"_index": "foo", "_type": "bar", "_id": "doc3", "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, - "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 1}] + "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] } ], "metric" : { "precisionatn": { "size": 10}} } - match: {rank_eval.quality_level: 1} - - match: {rank_eval.unknown_docs.amsterdam_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} - - match: {rank_eval.unknown_docs.berlin_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} + - match: {rank_eval.unknown_docs.amsterdam_query: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} + - match: {rank_eval.unknown_docs.berlin_query: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} --- "Reciprocal Rank": @@ -111,13 +111,13 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, # doc4 should be returned in third position, so reciprocal rank is 1/3 - "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 1}] + "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc4", "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 - "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 1}] + "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc4", "rating": 1}] } ], "metric" : { "reciprocal_rank": {} } @@ -134,13 +134,13 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, # doc4 should be returned in third position, so reciprocal rank is 1/3 - "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 1}] + "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc4", "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 - "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 1}] + "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc4", "rating": 1}] } ], "metric" : { diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml index 5df0b45fcc9..51f3393c21b 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml @@ -52,12 +52,12 @@ "id": "dcg_query", "request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] }, "ratings": [ - {"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 3}, - {"index": "foo", "type": "bar", "doc_id": "doc2", "rating": 2}, - {"index": "foo", "type": "bar", "doc_id": "doc3", "rating": 3}, - {"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 0}, - {"index": "foo", "type": "bar", "doc_id": "doc5", "rating": 1}, - {"index": "foo", "type": "bar", "doc_id": "doc6", "rating": 2}] + {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] } ], "metric" : { "dcg_at_n": { "size": 6}} @@ -75,12 +75,12 @@ "id": "dcg_query_reverse", "request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] }, "ratings": [ - {"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 3}, - {"index": "foo", "type": "bar", "doc_id": "doc2", "rating": 2}, - {"index": "foo", "type": "bar", "doc_id": "doc3", "rating": 3}, - {"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 0}, - {"index": "foo", "type": "bar", "doc_id": "doc5", "rating": 1}, - {"index": "foo", "type": "bar", "doc_id": "doc6", "rating": 2}] + {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] }, ], "metric" : { "dcg_at_n": { "size": 6}} @@ -98,23 +98,23 @@ "id": "dcg_query", "request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] }, "ratings": [ - {"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 3}, - {"index": "foo", "type": "bar", "doc_id": "doc2", "rating": 2}, - {"index": "foo", "type": "bar", "doc_id": "doc3", "rating": 3}, - {"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 0}, - {"index": "foo", "type": "bar", "doc_id": "doc5", "rating": 1}, - {"index": "foo", "type": "bar", "doc_id": "doc6", "rating": 2}] + {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] }, { "id": "dcg_query_reverse", "request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] }, "ratings": [ - {"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 3}, - {"index": "foo", "type": "bar", "doc_id": "doc2", "rating": 2}, - {"index": "foo", "type": "bar", "doc_id": "doc3", "rating": 3}, - {"index": "foo", "type": "bar", "doc_id": "doc4", "rating": 0}, - {"index": "foo", "type": "bar", "doc_id": "doc5", "rating": 1}, - {"index": "foo", "type": "bar", "doc_id": "doc6", "rating": 2}] + {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] }, ], "metric" : { "dcg_at_n": { "size": 6}} From 29402a28e0d1d8bfad13ec8e5e4dc58b7be36b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 22 Sep 2016 12:17:10 +0200 Subject: [PATCH 057/297] RankEval: Adding details section to response (#20497) In order to understand how well particular queries in a joint ranking evaluation request work we want to break down the overall metric into its components, each contributed by a particular query. The response structure now has a `details` section under which we can summarize this information. Each sub-section is keyed by the query-id and currently only contains the partial metric and the unknown_docs section for each query. --- .../rankeval/DiscountedCumulativeGainAt.java | 25 +++-- .../index/rankeval/EvalQueryQuality.java | 102 +++++++++++++++-- .../index/rankeval/MetricDetails.java | 27 +++++ .../index/rankeval/PrecisionAtN.java | 72 +++++++++++- .../index/rankeval/RankEvalPlugin.java | 14 ++- .../index/rankeval/RankEvalResponse.java | 58 ++++------ .../rankeval/RankedListQualityMetric.java | 4 +- .../index/rankeval/RatedDocumentKey.java | 14 ++- .../index/rankeval/ReciprocalRank.java | 60 +++++++++- .../rankeval/TransportRankEvalAction.java | 8 +- .../DiscountedCumulativeGainAtTests.java | 39 +++++-- .../index/rankeval/EvalQueryQualityTests.java | 105 ++++++++++++++++++ .../index/rankeval/PrecisionAtNTests.java | 33 ++++-- .../index/rankeval/RankEvalRequestTests.java | 8 +- .../index/rankeval/RankEvalResponseTests.java | 16 +-- .../index/rankeval/RankEvalSpecTests.java | 2 +- ...estHelper.java => RankEvalTestHelper.java} | 27 ++++- .../index/rankeval/RatedDocumentKeyTests.java | 40 ++++--- .../index/rankeval/RatedDocumentTests.java | 6 +- .../index/rankeval/RatedRequestsTests.java | 4 +- .../index/rankeval/ReciprocalRankTests.java | 24 ++-- .../test/rank_eval/10_basic.yaml | 28 ++++- .../rest-api-spec/test/rank_eval/20_dcg.yaml | 8 ++ 23 files changed, 571 insertions(+), 153 deletions(-) create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{XContentTestHelper.java => RankEvalTestHelper.java} (57%) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index 36d2a208353..b0a806287ca 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -30,8 +30,8 @@ import org.elasticsearch.search.SearchHit; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -139,13 +139,13 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { } @Override - public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { + public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { Map ratedDocsByKey = new HashMap<>(); for (RatedDocument doc : ratedDocs) { ratedDocsByKey.put(doc.getKey(), doc); } - Collection unknownDocIds = new ArrayList<>(); + List unknownDocIds = new ArrayList<>(); List ratings = new ArrayList<>(); for (int i = 0; (i < position && i < hits.length); i++) { RatedDocumentKey id = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); @@ -156,24 +156,29 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { unknownDocIds.add(id); if (unknownDocRating != null) { ratings.add(unknownDocRating); + } else { + // we add null here so that the later computation knows this position had no rating + ratings.add(null); } } } double dcg = computeDCG(ratings); if (normalize) { - Collections.sort(ratings, Collections.reverseOrder()); + Collections.sort(ratings, Comparator.nullsLast(Collections.reverseOrder())); double idcg = computeDCG(ratings); dcg = dcg / idcg; } - return new EvalQueryQuality(dcg, unknownDocIds); + return new EvalQueryQuality(taskId, dcg, unknownDocIds); } private static double computeDCG(List ratings) { int rank = 1; double dcg = 0; - for (int rating : ratings) { - dcg += (Math.pow(2, rating) - 1) / ((Math.log(rank + 1) / LOG2)); + for (Integer rating : ratings) { + if (rating != null) { + dcg += (Math.pow(2, rating) - 1) / ((Math.log(rank + 1) / LOG2)); + } rank++; } return dcg; @@ -208,7 +213,7 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { builder.endObject(); return builder; } - + @Override public final boolean equals(Object obj) { if (this == obj) { @@ -222,9 +227,11 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { Objects.equals(normalize, other.normalize) && Objects.equals(unknownDocRating, other.unknownDocRating); } - + @Override public final int hashCode() { return Objects.hash(position, normalize, unknownDocRating); } + + // TODO maybe also add debugging breakdown here } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index 9f8a192d6e7..58268522cc1 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -19,29 +19,107 @@ package org.elasticsearch.index.rankeval; -import java.util.Collection; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects;; /** - * Returned for each search specification. Summarizes the measured quality - * metric for this search request and adds the document ids found that were in - * the search result but not annotated in the original request. + * This class represents the partial information from running the ranking evaluation metric on one + * request alone. It contains all information necessary to render the response for this part of the + * overall evaluation. */ -public class EvalQueryQuality { +public class EvalQueryQuality implements ToXContent, Writeable { + + /** documents seen as result for one request that were not annotated.*/ + private List unknownDocs; + private String id; private double qualityLevel; + private MetricDetails optionalMetricDetails; - private Collection unknownDocs; - - public EvalQueryQuality (double qualityLevel, Collection unknownDocs) { - this.qualityLevel = qualityLevel; - this.unknownDocs = unknownDocs; + public EvalQueryQuality(String id, double qualityLevel, List unknownDocs) { + this.id = id; + this.unknownDocs = unknownDocs; + this.qualityLevel = qualityLevel; } - public Collection getUnknownDocs() { - return unknownDocs; + public EvalQueryQuality(StreamInput in) throws IOException { + this(in.readString(), in.readDouble(), in.readList(RatedDocumentKey::new)); + this.optionalMetricDetails = in.readOptionalNamedWriteable(MetricDetails.class); + } + + public String getId() { + return id; } public double getQualityLevel() { return qualityLevel; } + public List getUnknownDocs() { + return Collections.unmodifiableList(this.unknownDocs); + } + + public void addMetricDetails(MetricDetails breakdown) { + this.optionalMetricDetails = breakdown; + } + + public MetricDetails getMetricDetails() { + return this.optionalMetricDetails; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeDouble(qualityLevel); + out.writeVInt(unknownDocs.size()); + for (RatedDocumentKey key : unknownDocs) { + key.writeTo(out); + } + out.writeOptionalNamedWriteable(this.optionalMetricDetails); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(id); + builder.field("quality_level", this.qualityLevel); + builder.startArray("unknown_docs"); + for (RatedDocumentKey key : unknownDocs) { + key.toXContent(builder, params); + } + builder.endArray(); + if (optionalMetricDetails != null) { + builder.startObject("metric_details"); + optionalMetricDetails.toXContent(builder, params); + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + EvalQueryQuality other = (EvalQueryQuality) obj; + return Objects.equals(id, other.id) && + Objects.equals(qualityLevel, other.qualityLevel) && + Objects.equals(unknownDocs, other.unknownDocs) && + Objects.equals(optionalMetricDetails, other.optionalMetricDetails); + } + + @Override + public final int hashCode() { + return Objects.hash(id, qualityLevel, unknownDocs, optionalMetricDetails); + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java new file mode 100644 index 00000000000..af838111427 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.ToXContent; + +public interface MetricDetails extends ToXContent, NamedWriteable { + +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index ac8675a2a37..83b9de04f5d 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -120,7 +120,7 @@ public class PrecisionAtN extends RankedListQualityMetric { * @return precision at n for above {@link SearchResult} list. **/ @Override - public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { + public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { Collection relevantDocIds = new ArrayList<>(); Collection irrelevantDocIds = new ArrayList<>(); @@ -134,7 +134,7 @@ public class PrecisionAtN extends RankedListQualityMetric { int good = 0; int bad = 0; - Collection unknownDocIds = new ArrayList<>(); + List unknownDocIds = new ArrayList<>(); for (int i = 0; (i < n && i < hits.length); i++) { RatedDocumentKey hitKey = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); if (relevantDocIds.contains(hitKey)) { @@ -146,7 +146,9 @@ public class PrecisionAtN extends RankedListQualityMetric { } } double precision = (double) good / (good + bad); - return new EvalQueryQuality(precision, unknownDocIds); + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision, unknownDocIds); + evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(good, good + bad)); + return evalQueryQuality; } // TODO add abstraction that also works for other metrics @@ -194,9 +196,71 @@ public class PrecisionAtN extends RankedListQualityMetric { PrecisionAtN other = (PrecisionAtN) obj; return Objects.equals(n, other.n); } - + @Override public final int hashCode() { return Objects.hash(n); } + + public static class Breakdown implements MetricDetails { + + public static final String DOCS_RETRIEVED_FIELD = "docs_retrieved"; + public static final String RELEVANT_DOCS_RETRIEVED_FIELD = "relevant_docs_retrieved"; + private int relevantRetrieved; + private int retrieved; + + public Breakdown(int relevantRetrieved, int retrieved) { + this.relevantRetrieved = relevantRetrieved; + this.retrieved = retrieved; + } + + public Breakdown(StreamInput in) throws IOException { + this.relevantRetrieved = in.readVInt(); + this.retrieved = in.readVInt(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(RELEVANT_DOCS_RETRIEVED_FIELD, relevantRetrieved); + builder.field(DOCS_RETRIEVED_FIELD, retrieved); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(relevantRetrieved); + out.writeVInt(retrieved); + } + + @Override + public String getWriteableName() { + return NAME; + } + + public int getRelevantRetrieved() { + return relevantRetrieved; + } + + public int getRetrieved() { + return retrieved; + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PrecisionAtN.Breakdown other = (PrecisionAtN.Breakdown) obj; + return Objects.equals(relevantRetrieved, other.relevantRetrieved) && + Objects.equals(retrieved, other.retrieved); + } + + @Override + public final int hashCode() { + return Objects.hash(relevantRetrieved, retrieved); + } + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index bb59ff6051f..c741f59f74c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -49,9 +49,15 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin { */ @Override public List getNamedWriteables() { - List metrics = new ArrayList<>(); - metrics.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, PrecisionAtN.NAME, PrecisionAtN::new)); - metrics.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); - return metrics; + List namedWriteables = new ArrayList<>(); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, PrecisionAtN.NAME, PrecisionAtN::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGainAt.NAME, + DiscountedCumulativeGainAt::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, PrecisionAtN.NAME, + PrecisionAtN.Breakdown::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, ReciprocalRank.NAME, + ReciprocalRank.Breakdown::new)); + return namedWriteables; } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 43b66b46861..81994e86f24 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -26,8 +26,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -47,41 +46,37 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { /**Average precision observed when issuing query intents with this specification.*/ private double qualityLevel; /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ - private Map> unknownDocs; + private Map details; public RankEvalResponse() { } - public RankEvalResponse(double qualityLevel, Map> unknownDocs) { + public RankEvalResponse(double qualityLevel, Map partialResults) { this.qualityLevel = qualityLevel; - this.unknownDocs = unknownDocs; + this.details = partialResults; } public double getQualityLevel() { return qualityLevel; } - public Map> getUnknownDocs() { - return unknownDocs; + public Map getPartialResults() { + return Collections.unmodifiableMap(details); } @Override public String toString() { - return "RankEvalResponse, quality: " + qualityLevel + ", unknown docs: " + unknownDocs; + return "RankEvalResponse, quality: " + qualityLevel + ", partial results: " + details; } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeDouble(qualityLevel); - out.writeVInt(unknownDocs.size()); - for (String queryId : unknownDocs.keySet()) { + out.writeVInt(details.size()); + for (String queryId : details.keySet()) { out.writeString(queryId); - Collection collection = unknownDocs.get(queryId); - out.writeVInt(collection.size()); - for (RatedDocumentKey key : collection) { - key.writeTo(out); - } + details.get(queryId).writeTo(out); } } @@ -89,16 +84,12 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); this.qualityLevel = in.readDouble(); - int unknownDocumentSets = in.readVInt(); - this.unknownDocs = new HashMap<>(unknownDocumentSets); - for (int i = 0; i < unknownDocumentSets; i++) { + int partialResultSize = in.readVInt(); + this.details = new HashMap<>(partialResultSize); + for (int i = 0; i < partialResultSize; i++) { String queryId = in.readString(); - int numberUnknownDocs = in.readVInt(); - Collection collection = new ArrayList<>(numberUnknownDocs); - for (int d = 0; d < numberUnknownDocs; d++) { - collection.add(new RatedDocumentKey(in)); - } - this.unknownDocs.put(queryId, collection); + EvalQueryQuality partial = new EvalQueryQuality(in); + this.details.put(queryId, partial); } } @@ -106,18 +97,9 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("rank_eval"); builder.field("quality_level", qualityLevel); - builder.startObject("unknown_docs"); - for (String key : unknownDocs.keySet()) { - Collection keys = unknownDocs.get(key); - builder.startArray(key); - for (RatedDocumentKey docKey : keys) { - builder.startObject(); - builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), docKey.getIndex()); - builder.field(RatedDocument.TYPE_FIELD.getPreferredName(), docKey.getType()); - builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), docKey.getDocID()); - builder.endObject(); - } - builder.endArray(); + builder.startObject("details"); + for (String key : details.keySet()) { + details.get(key).toXContent(builder, params); } builder.endObject(); builder.endObject(); @@ -134,11 +116,11 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { } RankEvalResponse other = (RankEvalResponse) obj; return Objects.equals(qualityLevel, other.qualityLevel) && - Objects.equals(unknownDocs, other.unknownDocs); + Objects.equals(details, other.details); } @Override public final int hashCode() { - return Objects.hash(getClass(), qualityLevel, unknownDocs); + return Objects.hash(qualityLevel, details); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index e423ab3533c..a34cbd058cd 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -44,10 +44,12 @@ public abstract class RankedListQualityMetric extends ToXContentToBytes implemen * Returns a single metric representing the ranking quality of a set of returned documents * wrt. to a set of document Ids labeled as relevant for this search. * + * @param taskId the id of the query for which the ranking is currently evaluated * @param hits the result hits as returned by some search + * @param ratedDocs the documents that were ranked by human annotators for this query case * @return some metric representing the quality of the result hit list wrt. to relevant doc ids. * */ - public abstract EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs); + public abstract EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs); public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { RankedListQualityMetric rc; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java index ee08cf018da..35da907189f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -19,14 +19,16 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; import java.util.Objects; -public class RatedDocumentKey implements Writeable { +public class RatedDocumentKey extends ToXContentToBytes implements Writeable { private String docId; private String type; @@ -93,4 +95,14 @@ public class RatedDocumentKey implements Writeable { public final int hashCode() { return Objects.hash(index, type, docId); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), index); + builder.field(RatedDocument.TYPE_FIELD.getPreferredName(), type); + builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), docId); + builder.endObject(); + return builder; + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 803900a3521..c249706a948 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -30,7 +30,6 @@ import org.elasticsearch.search.SearchHit; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -114,7 +113,7 @@ public class ReciprocalRank extends RankedListQualityMetric { * @return reciprocal Rank for above {@link SearchResult} list. **/ @Override - public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { + public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { Set relevantDocIds = new HashSet<>(); Set irrelevantDocIds = new HashSet<>(); for (RatedDocument doc : ratedDocs) { @@ -125,7 +124,7 @@ public class ReciprocalRank extends RankedListQualityMetric { } } - Collection unknownDocIds = new ArrayList<>(); + List unknownDocIds = new ArrayList<>(); int firstRelevant = -1; boolean found = false; for (int i = 0; i < hits.length; i++) { @@ -141,7 +140,9 @@ public class ReciprocalRank extends RankedListQualityMetric { } double reciprocalRank = (firstRelevant == -1) ? 0 : 1.0d / firstRelevant; - return new EvalQueryQuality(reciprocalRank, unknownDocIds); + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, reciprocalRank, unknownDocIds); + evalQueryQuality.addMetricDetails(new Breakdown(firstRelevant)); + return evalQueryQuality; } @Override @@ -184,9 +185,58 @@ public class ReciprocalRank extends RankedListQualityMetric { ReciprocalRank other = (ReciprocalRank) obj; return Objects.equals(maxAcceptableRank, other.maxAcceptableRank); } - + @Override public final int hashCode() { return Objects.hash(maxAcceptableRank); } + + public static class Breakdown implements MetricDetails { + + private int firstRelevantRank; + + public Breakdown(int firstRelevantRank) { + this.firstRelevantRank = firstRelevantRank; + } + + public Breakdown(StreamInput in) throws IOException { + this.firstRelevantRank = in.readVInt(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("first_relevant", firstRelevantRank); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(firstRelevantRank); + } + + @Override + public String getWriteableName() { + return NAME; + } + + public int getFirstRelevantRank() { + return firstRelevantRank; + } + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ReciprocalRank.Breakdown other = (ReciprocalRank.Breakdown) obj; + return Objects.equals(firstRelevantRank, other.firstRelevantRank); + } + + @Override + public final int hashCode() { + return Objects.hash(firstRelevantRank); + } + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index f7f21eb81e7..65dbbc0ecc9 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -88,7 +88,6 @@ public class TransportRankEvalAction extends HandledTransportAction partialResults; private RankEvalSpec task; - private Map> unknownDocs; private AtomicInteger responseCounter; public RankEvalActionListener(ActionListener listener, RankEvalSpec task, RatedRequest specification, @@ -98,21 +97,20 @@ public class TransportRankEvalAction extends HandledTransportAction rated = new ArrayList<>(); - int[] relevanceRatings = new int[] { 3, 2, 3}; + Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1}; InternalSearchHit[] hits = new InternalSearchHit[6]; for (int i = 0; i < 6; i++) { if (i < relevanceRatings.length) { - rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); + if (relevanceRatings[i] != null) { + rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); + } } hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(6); - EvalQueryQuality result = dcg.evaluate(hits, rated); - assertEquals(12.392789260714371, result.getQualityLevel(), 0.00001); - assertEquals(3, result.getUnknownDocs().size()); + EvalQueryQuality result = dcg.evaluate("id", hits, rated); + assertEquals(12.779642067948913, result.getQualityLevel(), 0.00001); + assertEquals(2, result.getUnknownDocs().size()); + + /** + * Check with normalization: to get the maximal possible dcg, sort documents by relevance in descending order + * + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * ------------------------------------------------------------------------------------------- + * 1 | 3 | 7.0 | 1.0  | 7.0 + * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 + * 3 | 2 | 3.0 | 2.0  | 1.5 + * 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339 + * 5 | n.a | n.a | n.a.  | n.a. + * 6 | n.a | n.a | n.a  | n.a + * + * idcg = 13.347184833073591 (sum of last column) + */ + dcg.setNormalize(true); + assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); } public void testParseFromXContent() throws IOException { @@ -131,7 +150,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { } public void testXContentRoundtrip() throws IOException { DiscountedCumulativeGainAt testItem = createTestItem(); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); DiscountedCumulativeGainAt parsedItem = DiscountedCumulativeGainAt.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java new file mode 100644 index 00000000000..8924ff0f66d --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class EvalQueryQualityTests extends ESTestCase { + + private static NamedWriteableRegistry namedWritableRegistry = new NamedWriteableRegistry(new RankEvalPlugin().getNamedWriteables()); + + public static EvalQueryQuality randomEvalQueryQuality() { + List unknownDocs = new ArrayList<>(); + int numberOfUnknownDocs = randomInt(5); + for (int i = 0; i < numberOfUnknownDocs; i++) { + unknownDocs.add(RatedDocumentKeyTests.createRandomRatedDocumentKey()); + } + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAsciiOfLength(10), randomDoubleBetween(0.0, 1.0, true), unknownDocs); + if (randomBoolean()) { + // TODO randomize this + evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(1, 5)); + } + return evalQueryQuality; + } + + private static EvalQueryQuality copy(EvalQueryQuality original) throws IOException { + try (BytesStreamOutput output = new BytesStreamOutput()) { + original.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWritableRegistry)) { + return new EvalQueryQuality(in); + } + } + } + + public void testSerialization() throws IOException { + EvalQueryQuality original = randomEvalQueryQuality(); + EvalQueryQuality deserialized = copy(original); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + + public void testEqualsAndHash() throws IOException { + EvalQueryQuality testItem = randomEvalQueryQuality(); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), + copy(testItem)); + } + + private static EvalQueryQuality mutateTestItem(EvalQueryQuality original) { + String id = original.getId(); + double qualityLevel = original.getQualityLevel(); + List unknownDocs = original.getUnknownDocs(); + MetricDetails breakdown = original.getMetricDetails(); + switch (randomIntBetween(0, 3)) { + case 0: + id = id + "_"; + break; + case 1: + qualityLevel = qualityLevel + 0.1; + break; + case 2: + unknownDocs = new ArrayList<>(unknownDocs); + unknownDocs.add(RatedDocumentKeyTests.createRandomRatedDocumentKey()); + break; + case 3: + if (breakdown == null) { + breakdown = new PrecisionAtN.Breakdown(1, 5); + } else { + breakdown = null; + } + break; + default: + throw new IllegalStateException("The test should only allow three parameters mutated"); + } + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(id, qualityLevel, unknownDocs); + evalQueryQuality.addMetricDetails(breakdown); + return evalQueryQuality; + } + + +} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index bc78484832f..e892126a37a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -46,7 +46,10 @@ public class PrecisionAtNTests extends ESTestCase { InternalSearchHit[] hits = new InternalSearchHit[1]; hits[0] = new InternalSearchHit(0, "0", new Text("testtype"), Collections.emptyMap()); hits[0].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); - assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + assertEquals(1, evaluated.getQualityLevel(), 0.00001); + assertEquals(1, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(1, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException { @@ -61,7 +64,10 @@ public class PrecisionAtNTests extends ESTestCase { hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } - assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + assertEquals((double) 4 / 5, evaluated.getQualityLevel(), 0.00001); + assertEquals(4, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(5, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } /** @@ -82,7 +88,10 @@ public class PrecisionAtNTests extends ESTestCase { } PrecisionAtN precisionAtN = new PrecisionAtN(5); precisionAtN.setRelevantRatingThreshhold(2); - assertEquals((double) 3 / 5, precisionAtN.evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluated = precisionAtN.evaluate("id", hits, rated); + assertEquals((double) 3 / 5, evaluated.getQualityLevel(), 0.00001); + assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(5, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveCorrectIndex() throws IOException, InterruptedException, ExecutionException { @@ -97,7 +106,10 @@ public class PrecisionAtNTests extends ESTestCase { hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } - assertEquals((double) 2 / 3, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); + assertEquals(2, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveCorrectType() throws IOException, InterruptedException, ExecutionException { @@ -112,7 +124,10 @@ public class PrecisionAtNTests extends ESTestCase { hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } - assertEquals((double) 2 / 3, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); + assertEquals(2, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testParseFromXContent() throws IOException { @@ -129,9 +144,9 @@ public class PrecisionAtNTests extends ESTestCase { public void testCombine() { PrecisionAtN metric = new PrecisionAtN(); Vector partialResults = new Vector<>(3); - partialResults.add(new EvalQueryQuality(0.1, emptyList())); - partialResults.add(new EvalQueryQuality(0.2, emptyList())); - partialResults.add(new EvalQueryQuality(0.6, emptyList())); + partialResults.add(new EvalQueryQuality("a", 0.1, emptyList())); + partialResults.add(new EvalQueryQuality("b", 0.2, emptyList())); + partialResults.add(new EvalQueryQuality("c", 0.6, emptyList())); assertEquals(0.3, metric.combine(partialResults), Double.MIN_VALUE); } @@ -142,7 +157,7 @@ public class PrecisionAtNTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { PrecisionAtN testItem = createTestItem(); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); PrecisionAtN parsedItem = PrecisionAtN.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index 9fee4685e6c..b6852be8311 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -83,14 +83,14 @@ public class RankEvalRequestTests extends ESIntegTestCase { RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); assertEquals(1.0, response.getQualityLevel(), Double.MIN_VALUE); - Set>> entrySet = response.getUnknownDocs().entrySet(); + Set> entrySet = response.getPartialResults().entrySet(); assertEquals(2, entrySet.size()); - for (Entry> entry : entrySet) { + for (Entry entry : entrySet) { if (entry.getKey() == "amsterdam_query") { - assertEquals(2, entry.getValue().size()); + assertEquals(2, entry.getValue().getUnknownDocs().size()); } if (entry.getKey() == "berlin_query") { - assertEquals(5, entry.getValue().size()); + assertEquals(5, entry.getValue().getUnknownDocs().size()); } } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index b5b68ec22ae..73ba2301d58 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,17 +36,18 @@ import java.util.Map; public class RankEvalResponseTests extends ESTestCase { private static RankEvalResponse createRandomResponse() { - Map> unknownDocs = new HashMap<>(); - int numberOfSets = randomIntBetween(0, 5); - for (int i = 0; i < numberOfSets; i++) { - List ids = new ArrayList<>(); + int numberOfRequests = randomIntBetween(0, 5); + Map partials = new HashMap<>(numberOfRequests); + for (int i = 0; i < numberOfRequests; i++) { + String id = randomAsciiOfLengthBetween(3, 10); int numberOfUnknownDocs = randomIntBetween(0, 5); + List unknownDocs = new ArrayList<>(numberOfUnknownDocs); for (int d = 0; d < numberOfUnknownDocs; d++) { - ids.add(new RatedDocumentKey(randomAsciiOfLength(5), randomAsciiOfLength(5), randomAsciiOfLength(5))); + unknownDocs.add(RatedDocumentKeyTests.createRandomRatedDocumentKey()); } - unknownDocs.put(randomAsciiOfLength(5), ids); + partials.put(id, new EvalQueryQuality(id, randomDoubleBetween(0.0, 1.0, true), unknownDocs)); } - return new RankEvalResponse(randomDouble(), unknownDocs ); + return new RankEvalResponse(randomDouble(), partials); } public void testSerialization() throws IOException { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 97fd0a9f281..59c17d71d32 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -89,7 +89,7 @@ public class RankEvalSpecTests extends ESTestCase { RankEvalSpec testItem = new RankEvalSpec(specs, metric); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentTestHelper.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java similarity index 57% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentTestHelper.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java index e02772ad6f6..4218ae5d5cd 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentTestHelper.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java @@ -30,9 +30,16 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; -public class XContentTestHelper { +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; - public static XContentParser roundtrip(ToXContentToBytes testItem) throws IOException { +public class RankEvalTestHelper { + + public static XContentParser roundtrip(ToXContentToBytes testItem) throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(ESTestCase.randomFrom(XContentType.values())); if (ESTestCase.randomBoolean()) { builder.prettyPrint(); @@ -42,4 +49,20 @@ public class XContentTestHelper { XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); return itemParser; } + + public static void testHashCodeAndEquals(Object testItem, Object mutation, Object secondCopy) { + assertFalse("testItem is equal to null", testItem.equals(null)); + assertFalse("testItem is equal to incompatible type", testItem.equals("")); + assertTrue("testItem is not equal to self", testItem.equals(testItem)); + assertThat("same testItem's hashcode returns different values if called multiple times", testItem.hashCode(), + equalTo(testItem.hashCode())); + + assertThat("different testItem should not be equal", mutation, not(equalTo(testItem))); + + assertNotSame("testItem copy is not same as original", testItem, secondCopy); + assertTrue("testItem is not equal to its copy", testItem.equals(secondCopy)); + assertTrue("equals is not symmetric", secondCopy.equals(testItem)); + assertThat("testItem copy's hashcode is different from original hashcode", secondCopy.hashCode(), + equalTo(testItem.hashCode())); + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java index 26661c086a6..231a1b8e586 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java @@ -23,45 +23,43 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; public class RatedDocumentKeyTests extends ESTestCase { - public void testEqualsAndHash() throws IOException { + static RatedDocumentKey createRandomRatedDocumentKey() { String index = randomAsciiOfLengthBetween(0, 10); String type = randomAsciiOfLengthBetween(0, 10); String docId = randomAsciiOfLengthBetween(0, 10); + return new RatedDocumentKey(index, type, docId); + } - RatedDocumentKey testItem = new RatedDocumentKey(index, type, docId); + public RatedDocumentKey createRandomTestItem() { + return createRandomRatedDocumentKey(); + } - assertFalse("key is equal to null", testItem.equals(null)); - assertFalse("key is equal to incompatible type", testItem.equals("")); - assertTrue("key is not equal to self", testItem.equals(testItem)); - assertThat("same key's hashcode returns different values if called multiple times", testItem.hashCode(), - equalTo(testItem.hashCode())); - - RatedDocumentKey mutation; + public RatedDocumentKey mutateTestItem(RatedDocumentKey original) { + String index = original.getIndex(); + String type = original.getType(); + String docId = original.getDocID(); switch (randomIntBetween(0, 2)) { case 0: - mutation = new RatedDocumentKey(testItem.getIndex() + "_foo", testItem.getType(), testItem.getDocID()); + index = index + "_"; break; case 1: - mutation = new RatedDocumentKey(testItem.getIndex(), testItem.getType() + "_foo", testItem.getDocID()); + type = type + "_"; break; case 2: - mutation = new RatedDocumentKey(testItem.getIndex(), testItem.getType(), testItem.getDocID() + "_foo"); + docId = docId + "_"; break; default: throw new IllegalStateException("The test should only allow three parameters mutated"); } + return new RatedDocumentKey(index, type, docId); + } - assertThat("different keys should not be equal", mutation, not(equalTo(testItem))); - - RatedDocumentKey secondEqualKey = new RatedDocumentKey(index, type, docId); - assertTrue("key is not equal to its copy", testItem.equals(secondEqualKey)); - assertTrue("equals is not symmetric", secondEqualKey.equals(testItem)); - assertThat("key copy's hashcode is different from original hashcode", secondEqualKey.hashCode(), - equalTo(testItem.hashCode())); + public void testEqualsAndHash() throws IOException { + RatedDocumentKey testItem = createRandomRatedDocumentKey(); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), + new RatedDocumentKey(testItem.getIndex(), testItem.getType(), testItem.getDocID())); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index b3af34423c7..33266b897f7 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -27,7 +27,7 @@ import java.io.IOException; public class RatedDocumentTests extends ESTestCase { - public static RatedDocument createTestItem() { + public static RatedDocument createRatedDocument() { String index = randomAsciiOfLength(10); String type = randomAsciiOfLength(10); String docId = randomAsciiOfLength(10); @@ -37,8 +37,8 @@ public class RatedDocumentTests extends ESTestCase { } public void testXContentParsing() throws IOException { - RatedDocument testItem = createTestItem(); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + RatedDocument testItem = createRatedDocument(); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); RatedDocument parsedItem = RatedDocument.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 0d1615a5d05..c325181727e 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -76,7 +76,7 @@ public class RatedRequestsTests extends ESTestCase { List ratedDocs = new ArrayList<>(); int size = randomIntBetween(0, 2); for (int i = 0; i < size; i++) { - ratedDocs.add(RatedDocumentTests.createTestItem()); + ratedDocs.add(RatedDocumentTests.createRatedDocument()); } return new RatedRequest(specId, testRequest, indices, types, ratedDocs); @@ -96,7 +96,7 @@ public class RatedRequestsTests extends ESTestCase { } RatedRequest testItem = createTestItem(indices, types); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index c3f951b09b4..6dc5e670db5 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -63,17 +63,19 @@ public class ReciprocalRankTests extends ESTestCase { } int rankAtFirstRelevant = relevantAt + 1; - EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); + EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); if (rankAtFirstRelevant <= maxRank) { assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE); + assertEquals(rankAtFirstRelevant, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); // check that if we lower maxRank by one, we don't find any result and get 0.0 quality level reciprocalRank = new ReciprocalRank(rankAtFirstRelevant - 1); - evaluation = reciprocalRank.evaluate(hits, ratedDocs); + evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); } else { assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); + assertEquals(-1, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } } @@ -95,8 +97,9 @@ public class ReciprocalRankTests extends ESTestCase { } } - EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); + EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE); + assertEquals(relevantAt + 1, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } /** @@ -119,15 +122,17 @@ public class ReciprocalRankTests extends ESTestCase { ReciprocalRank reciprocalRank = new ReciprocalRank(); reciprocalRank.setRelevantRatingThreshhold(2); - assertEquals((double) 1 / 3, reciprocalRank.evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, rated); + assertEquals((double) 1 / 3, evaluation.getQualityLevel(), 0.00001); + assertEquals(3, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } public void testCombine() { ReciprocalRank reciprocalRank = new ReciprocalRank(); Vector partialResults = new Vector<>(3); - partialResults.add(new EvalQueryQuality(0.5, emptyList())); - partialResults.add(new EvalQueryQuality(1.0, emptyList())); - partialResults.add(new EvalQueryQuality(0.75, emptyList())); + partialResults.add(new EvalQueryQuality("id1", 0.5, emptyList())); + partialResults.add(new EvalQueryQuality("id2", 1.0, emptyList())); + partialResults.add(new EvalQueryQuality("id3", 0.75, emptyList())); assertEquals(0.75, reciprocalRank.combine(partialResults), Double.MIN_VALUE); } @@ -139,7 +144,7 @@ public class ReciprocalRankTests extends ESTestCase { hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } List ratedDocs = new ArrayList<>(); - EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); + EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); } @@ -147,7 +152,7 @@ public class ReciprocalRankTests extends ESTestCase { int position = randomIntBetween(0, 1000); ReciprocalRank testItem = new ReciprocalRank(position); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); ReciprocalRank parsedItem = ReciprocalRank.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); @@ -155,5 +160,4 @@ public class ReciprocalRankTests extends ESTestCase { assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); } - } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 2abad7353e9..63944f899f1 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -60,8 +60,13 @@ } - match: {rank_eval.quality_level: 1} - - match: {rank_eval.unknown_docs.amsterdam_query: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} - - match: {rank_eval.unknown_docs.berlin_query: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} + - match: {rank_eval.details.amsterdam_query.quality_level: 1.0} + - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} + - match: {rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}} + - match: {rank_eval.details.berlin_query.quality_level: 1.0} + - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} + - match: {rank_eval.details.berlin_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}} + --- "Reciprocal Rank": @@ -105,7 +110,7 @@ - do: rank_eval: - body: { + body: { "requests" : [ { "id": "amsterdam_query", @@ -125,6 +130,13 @@ # average is (1/3 + 1/2)/2 = 5/12 ~ 0.41666666666666663 - match: {rank_eval.quality_level: 0.41666666666666663} + - match: {rank_eval.details.amsterdam_query.quality_level: 0.3333333333333333} + - match: {rank_eval.details.amsterdam_query.metric_details: {"first_relevant": 3}} + - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc2"}, + {"_index": "foo", "_type": "bar", "_id": "doc3"} ]} + - match: {rank_eval.details.berlin_query.quality_level: 0.5} + - match: {rank_eval.details.berlin_query.metric_details: {"first_relevant": 2}} + - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc1"}]} - do: rank_eval: @@ -145,10 +157,18 @@ ], "metric" : { "reciprocal_rank": { + # the following will make the first query have a quality value of 0.0 "max_acceptable_rank" : 2 } } } # average is (0 + 1/2)/2 = 1/4 - - match: {rank_eval.quality_level: 0.25} + - match: {rank_eval.quality_level: 0.25} + - match: {rank_eval.details.amsterdam_query.quality_level: 0} + - match: {rank_eval.details.amsterdam_query.metric_details: {"first_relevant": -1}} + - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc2"}, + {"_index": "foo", "_type": "bar", "_id": "doc3"} ]} + - match: {rank_eval.details.berlin_query.quality_level: 0.5} + - match: {rank_eval.details.berlin_query.metric_details: {"first_relevant": 2}} + - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc1"}]} diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml index 51f3393c21b..8310389c02a 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml @@ -64,6 +64,8 @@ } - match: {rank_eval.quality_level: 13.84826362927298} + - match: {rank_eval.details.dcg_query.quality_level: 13.84826362927298} + - match: {rank_eval.details.dcg_query.unknown_docs: [ ]} # reverse the order in which the results are returned (less relevant docs first) @@ -87,6 +89,8 @@ } - match: {rank_eval.quality_level: 10.29967439154499} + - match: {rank_eval.details.dcg_query_reverse.quality_level: 10.29967439154499} + - match: {rank_eval.details.dcg_query_reverse.unknown_docs: [ ]} # if we mix both, we should get the average @@ -121,3 +125,7 @@ } - match: {rank_eval.quality_level: 12.073969010408984} + - match: {rank_eval.details.dcg_query.quality_level: 13.84826362927298} + - match: {rank_eval.details.dcg_query.unknown_docs: [ ]} + - match: {rank_eval.details.dcg_query_reverse.quality_level: 10.29967439154499} + - match: {rank_eval.details.dcg_query_reverse.unknown_docs: [ ]} From c3380863bea406c8e3f78bd841f3844be76e7e3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 10 Oct 2016 12:57:18 +0200 Subject: [PATCH 058/297] Adapting RestRankEvalAction to changes in API on master --- .../elasticsearch/index/rankeval/RestRankEvalAction.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index aebede0ff50..3a629f94b45 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; @@ -173,7 +172,7 @@ public class RestRankEvalAction extends BaseRestHandler { } @Override - public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws IOException { + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { RankEvalRequest rankEvalRequest = new RankEvalRequest(); BytesReference restContent = RestActions.hasBodyContent(request) ? RestActions.getRestContent(request) : null; try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { @@ -184,7 +183,8 @@ public class RestRankEvalAction extends BaseRestHandler { new RankEvalContext(parseFieldMatcher, parseContext, searchRequestParsers)); } } - client.execute(RankEvalAction.INSTANCE, rankEvalRequest, new RestToXContentListener(channel)); + return channel -> client.executeLocally(RankEvalAction.INSTANCE, rankEvalRequest, + new RestToXContentListener(channel)); } public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context) From ebe13100df634b47b9ae93b6d59ad01b686294d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 21 Sep 2016 14:12:47 +0200 Subject: [PATCH 059/297] Add `hits` section to response for each ranking evaluation query This change adds a `hits` section to the response part for each ranking evaluation query, containing a list of documents (index/type/id) and ratings (if the document was rated in the request). This section can be used to better understand the calculation of the ranking quality of this particular query, but it can also be used to identify the "unknown" (that is unrated) documents that were part of the seach hits, for example because a UI later wants to present those documents to the user to get a rating for them. If the user specifies a set of field names using a parameter called `summary_fields` in the request, those fields are also included as part of the response in addition to "_index", "_type", "_id". --- .../rankeval/DiscountedCumulativeGainAt.java | 27 +++-- .../index/rankeval/EvalQueryQuality.java | 35 ++++-- .../index/rankeval/PrecisionAtN.java | 23 ++-- .../index/rankeval/RatedRequest.java | 32 ++++- .../index/rankeval/RatedSearchHit.java | 114 ++++++++++++++++++ .../index/rankeval/ReciprocalRank.java | 24 +++- .../rankeval/TransportRankEvalAction.java | 8 ++ .../DiscountedCumulativeGainAtTests.java | 55 ++++++++- .../index/rankeval/EvalQueryQualityTests.java | 34 +++--- .../index/rankeval/RankEvalRequestTests.java | 38 +++++- .../index/rankeval/RankEvalTestHelper.java | 36 +++++- .../index/rankeval/RatedRequestsTests.java | 11 +- .../index/rankeval/RatedSearchHitTests.java | 71 +++++++++++ .../test/rank_eval/10_basic.yaml | 32 +++-- 14 files changed, 472 insertions(+), 68 deletions(-) create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index b0a806287ca..d4728f81fef 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -36,6 +36,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; public class DiscountedCumulativeGainAt extends RankedListQualityMetric { @@ -140,36 +141,44 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { @Override public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { - Map ratedDocsByKey = new HashMap<>(); + Map ratedDocsByKey = new HashMap<>(ratedDocs.size()); + List allRatings = new ArrayList<>(ratedDocs.size()); for (RatedDocument doc : ratedDocs) { ratedDocsByKey.put(doc.getKey(), doc); + allRatings.add(doc.getRating()); } List unknownDocIds = new ArrayList<>(); - List ratings = new ArrayList<>(); + List hitsAndRatings = new ArrayList<>(); + List ratingsInSearchHits = new ArrayList<>(); for (int i = 0; (i < position && i < hits.length); i++) { RatedDocumentKey id = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); RatedDocument ratedDoc = ratedDocsByKey.get(id); if (ratedDoc != null) { - ratings.add(ratedDoc.getRating()); + ratingsInSearchHits.add(ratedDoc.getRating()); + hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.of(ratedDoc.getRating()))); } else { unknownDocIds.add(id); if (unknownDocRating != null) { - ratings.add(unknownDocRating); + ratingsInSearchHits.add(unknownDocRating); + hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.of(unknownDocRating))); } else { // we add null here so that the later computation knows this position had no rating - ratings.add(null); + ratingsInSearchHits.add(null); + hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.empty())); } } } - double dcg = computeDCG(ratings); + double dcg = computeDCG(ratingsInSearchHits); if (normalize) { - Collections.sort(ratings, Comparator.nullsLast(Collections.reverseOrder())); - double idcg = computeDCG(ratings); + Collections.sort(allRatings, Comparator.nullsLast(Collections.reverseOrder())); + double idcg = computeDCG(allRatings.subList(0, Math.min(hits.length, allRatings.size()))); dcg = dcg / idcg; } - return new EvalQueryQuality(taskId, dcg, unknownDocIds); + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, dcg, unknownDocIds); + evalQueryQuality.addHitsAndRatings(hitsAndRatings); + return evalQueryQuality; } private static double computeDCG(List ratings) { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index 58268522cc1..caa0ee7b179 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects;; @@ -42,6 +43,7 @@ public class EvalQueryQuality implements ToXContent, Writeable { private String id; private double qualityLevel; private MetricDetails optionalMetricDetails; + private List hits = new ArrayList<>(); public EvalQueryQuality(String id, double qualityLevel, List unknownDocs) { this.id = id; @@ -51,9 +53,19 @@ public class EvalQueryQuality implements ToXContent, Writeable { public EvalQueryQuality(StreamInput in) throws IOException { this(in.readString(), in.readDouble(), in.readList(RatedDocumentKey::new)); + this.hits = in.readList(RatedSearchHit::new); this.optionalMetricDetails = in.readOptionalNamedWriteable(MetricDetails.class); } + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeDouble(qualityLevel); + out.writeList(unknownDocs); + out.writeList(hits); + out.writeOptionalNamedWriteable(this.optionalMetricDetails); + } + public String getId() { return id; } @@ -74,15 +86,12 @@ public class EvalQueryQuality implements ToXContent, Writeable { return this.optionalMetricDetails; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(id); - out.writeDouble(qualityLevel); - out.writeVInt(unknownDocs.size()); - for (RatedDocumentKey key : unknownDocs) { - key.writeTo(out); - } - out.writeOptionalNamedWriteable(this.optionalMetricDetails); + public void addHitsAndRatings(List hits) { + this.hits = hits; + } + + public List getHitsAndRatings() { + return this.hits; } @Override @@ -94,6 +103,11 @@ public class EvalQueryQuality implements ToXContent, Writeable { key.toXContent(builder, params); } builder.endArray(); + builder.startArray("hits"); + for (RatedSearchHit hit : hits) { + hit.toXContent(builder, params); + } + builder.endArray(); if (optionalMetricDetails != null) { builder.startObject("metric_details"); optionalMetricDetails.toXContent(builder, params); @@ -115,11 +129,12 @@ public class EvalQueryQuality implements ToXContent, Writeable { return Objects.equals(id, other.id) && Objects.equals(qualityLevel, other.qualityLevel) && Objects.equals(unknownDocs, other.unknownDocs) && + Objects.equals(hits, other.hits) && Objects.equals(optionalMetricDetails, other.optionalMetricDetails); } @Override public final int hashCode() { - return Objects.hash(id, qualityLevel, unknownDocs, optionalMetricDetails); + return Objects.hash(id, qualityLevel, unknownDocs, hits, optionalMetricDetails); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 83b9de04f5d..ecf277037a0 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -30,9 +30,11 @@ import org.elasticsearch.search.SearchHit; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Optional; import javax.naming.directory.SearchResult; @@ -122,32 +124,39 @@ public class PrecisionAtN extends RankedListQualityMetric { @Override public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { - Collection relevantDocIds = new ArrayList<>(); - Collection irrelevantDocIds = new ArrayList<>(); + Map relevantDocIds = new HashMap<>(); + Map irrelevantDocIds = new HashMap<>(); for (RatedDocument doc : ratedDocs) { if (doc.getRating() >= this.relevantRatingThreshhold) { - relevantDocIds.add(doc.getKey()); + relevantDocIds.put(doc.getKey(), doc); } else { - irrelevantDocIds.add(doc.getKey()); + irrelevantDocIds.put(doc.getKey(), doc); } } int good = 0; int bad = 0; List unknownDocIds = new ArrayList<>(); + List hitsAndRatings = new ArrayList<>(); for (int i = 0; (i < n && i < hits.length); i++) { RatedDocumentKey hitKey = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); - if (relevantDocIds.contains(hitKey)) { + if (relevantDocIds.keySet().contains(hitKey)) { + RatedDocument ratedDocument = relevantDocIds.get(hitKey); good++; - } else if (irrelevantDocIds.contains(hitKey)) { + hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.of(ratedDocument.getRating()))); + } else if (irrelevantDocIds.keySet().contains(hitKey)) { + RatedDocument ratedDocument = irrelevantDocIds.get(hitKey); bad++; + hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.of(ratedDocument.getRating()))); } else { unknownDocIds.add(hitKey); + hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.empty())); } } double precision = (double) good / (good + bad); EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision, unknownDocIds); evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(good, good + bad)); + evalQueryQuality.addHitsAndRatings(hitsAndRatings); return evalQueryQuality; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index f40f004bad7..dbfc449b7ea 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -47,6 +47,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { private SearchSourceBuilder testRequest; private List indices = new ArrayList<>(); private List types = new ArrayList<>(); + private List summaryFields = new ArrayList<>(); /** Collection of rated queries for this query QA specification.*/ private List ratedDocs = new ArrayList<>(); @@ -82,6 +83,11 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { for (int i = 0; i < intentSize; i++) { ratedDocs.add(new RatedDocument(in)); } + int summaryFieldsSize = in.readInt(); + summaryFields = new ArrayList<>(summaryFieldsSize); + for (int i = 0; i < summaryFieldsSize; i++) { + this.summaryFields.add(in.readString()); + } } @Override @@ -100,6 +106,10 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { for (RatedDocument ratedDoc : ratedDocs) { ratedDoc.writeTo(out); } + out.writeInt(summaryFields.size()); + for (String fieldName : summaryFields) { + out.writeString(fieldName); + } } public SearchSourceBuilder getTestRequest() { @@ -146,13 +156,24 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { this.ratedDocs = ratedDocs; } + public void setSummaryFields(List fields) { + this.summaryFields = fields; + } + + /** Returns a list of fields that are included in the docs summary of matched documents. */ + public List getSummaryFields() { + return summaryFields; + } + private static final ParseField ID_FIELD = new ParseField("id"); private static final ParseField REQUEST_FIELD = new ParseField("request"); private static final ParseField RATINGS_FIELD = new ParseField("ratings"); + private static final ParseField FIELDS_FIELD = new ParseField("summary_fields"); private static final ObjectParser PARSER = new ObjectParser<>("requests", RatedRequest::new); static { PARSER.declareString(RatedRequest::setSpecId, ID_FIELD); + PARSER.declareStringArray(RatedRequest::setSummaryFields, FIELDS_FIELD); PARSER.declareObject(RatedRequest::setTestRequest, (p, c) -> { try { return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters(), c.getSearchExtParsers()); @@ -186,6 +207,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { * }, * "size": 10 * }, + * "summary_fields" : ["body"], * "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ] * } */ @@ -203,6 +225,13 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { doc.toXContent(builder, params); } builder.endArray(); + if (this.summaryFields.isEmpty() == false) { + builder.startArray(FIELDS_FIELD.getPreferredName()); + for (String field : this.summaryFields) { + builder.value(field); + } + builder.endArray(); + } builder.endObject(); return builder; } @@ -220,11 +249,12 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { Objects.equals(testRequest, other.testRequest) && Objects.equals(indices, other.indices) && Objects.equals(types, other.types) && + Objects.equals(summaryFields, summaryFields) && Objects.equals(ratedDocs, other.ratedDocs); } @Override public final int hashCode() { - return Objects.hash(specId, testRequest, indices.hashCode(), types.hashCode(), ratedDocs.hashCode()); + return Objects.hash(specId, testRequest, indices, types, summaryFields, ratedDocs); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java new file mode 100644 index 00000000000..b2aaceeb34c --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java @@ -0,0 +1,114 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.internal.InternalSearchHit; + +import java.io.IOException; +import java.util.Objects; +import java.util.Optional; + +public class RatedSearchHit implements Writeable, ToXContent { + + private final SearchHit searchHit; + private final Optional rating; + + public RatedSearchHit(SearchHit searchHit, Optional rating) { + this.searchHit = searchHit; + this.rating = rating; + } + + public RatedSearchHit(StreamInput in) throws IOException { + this(InternalSearchHit.readSearchHit(in), in.readBoolean() == true ? Optional.of(in.readVInt()) : Optional.empty()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + searchHit.writeTo(out); + out.writeBoolean(rating.isPresent()); + if (rating.isPresent()) { + out.writeVInt(rating.get()); + } + } + + public SearchHit getSearchHit() { + return this.searchHit; + } + + public Optional getRating() { + return this.rating; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.field("hit", (ToXContent) searchHit); + builder.field("rating", rating.orElse(null)); + builder.endObject(); + return builder; + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + RatedSearchHit other = (RatedSearchHit) obj; + // NORELEASE this is a workaround because InternalSearchHit does not properly implement equals()/hashCode(), so we compare their + // xcontent + XContentBuilder builder; + String hitAsXContent; + String otherHitAsXContent; + try { + builder = XContentFactory.jsonBuilder(); + hitAsXContent = searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS).string(); + builder = XContentFactory.jsonBuilder(); + otherHitAsXContent = other.searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS).string(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return Objects.equals(rating, other.rating) && + Objects.equals(hitAsXContent, otherHitAsXContent); + } + + @Override + public final int hashCode() { + //NORELEASE for this to work requires InternalSearchHit to properly implement equals()/hashCode() + XContentBuilder builder; + String hitAsXContent; + try { + builder = XContentFactory.jsonBuilder(); + hitAsXContent = searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS).string(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return Objects.hash(rating, hitAsXContent); + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index c249706a948..ce892b6ba6c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -30,9 +30,12 @@ import org.elasticsearch.search.SearchHit; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import javax.naming.directory.SearchResult; @@ -114,6 +117,11 @@ public class ReciprocalRank extends RankedListQualityMetric { **/ @Override public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { + Map ratedDocsByKey = new HashMap<>(); + for (RatedDocument doc : ratedDocs) { + ratedDocsByKey.put(doc.getKey(), doc); + } + Set relevantDocIds = new HashSet<>(); Set irrelevantDocIds = new HashSet<>(); for (RatedDocument doc : ratedDocs) { @@ -125,23 +133,31 @@ public class ReciprocalRank extends RankedListQualityMetric { } List unknownDocIds = new ArrayList<>(); + List hitsAndRatings = new ArrayList<>(); int firstRelevant = -1; boolean found = false; for (int i = 0; i < hits.length; i++) { RatedDocumentKey key = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); - if (relevantDocIds.contains(key)) { - if (found == false && i < maxAcceptableRank) { - firstRelevant = i + 1; // add one because rank is not 0-based - found = true; + RatedDocument ratedDocument = ratedDocsByKey.get(key); + if (ratedDocument != null) { + if (ratedDocument.getRating() >= this.relevantRatingThreshhold) { + if (found == false && i < maxAcceptableRank) { + firstRelevant = i + 1; // add one because rank is not + // 0-based + found = true; + } + hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.of(ratedDocument.getRating()))); } } else { unknownDocIds.add(key); + hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.empty())); } } double reciprocalRank = (firstRelevant == -1) ? 0 : 1.0d / firstRelevant; EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, reciprocalRank, unknownDocIds); evalQueryQuality.addMetricDetails(new Breakdown(firstRelevant)); + evalQueryQuality.addHitsAndRatings(hitsAndRatings); return evalQueryQuality; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 65dbbc0ecc9..ee5763da3c7 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -72,6 +73,13 @@ public class TransportRankEvalAction extends HandledTransportAction summaryFields = querySpecification.getSummaryFields(); + if (summaryFields.isEmpty()) { + specRequest.fetchSource(false); + } else { + specRequest.fetchSource(summaryFields.toArray(new String[summaryFields.size()]), new String[0]); + } + String[] indices = new String[querySpecification.getIndices().size()]; querySpecification.getIndices().toArray(indices); SearchRequest templatedRequest = new SearchRequest(indices, specRequest); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java index eac725bba93..62f778c3204 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java @@ -50,7 +50,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { * * dcg = 13.84826362927298 (sum of last column) */ - public void testDCGAtSix() throws IOException, InterruptedException, ExecutionException { + public void testDCGAt() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); int[] relevanceRatings = new int[] { 3, 2, 3, 0, 1, 2 }; InternalSearchHit[] hits = new InternalSearchHit[6]; @@ -130,6 +130,59 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); } + /** + * This tests that normalization works as expected when there are more rated documents than search hits + * because we restrict DCG to be calculated at the fourth position + * + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * ------------------------------------------------------------------------------------------- + * 1 | 3 | 7.0 | 1.0 | 7.0 + * 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 + * 3 | 3 | 7.0 | 2.0 | 3.5 + * 4 | n/a | n/a | n/a | n/a + * ----------------------------------------------------------------- + * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163 + * 6 | n/a | n/a | n/a | n/a + * + * dcg = 12.392789260714371 (sum of last column until position 4) + */ + public void testDCGAtFourMoreRatings() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1}; + InternalSearchHit[] hits = new InternalSearchHit[6]; + for (int i = 0; i < 6; i++) { + if (i < relevanceRatings.length) { + if (relevanceRatings[i] != null) { + rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); + } + } + hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); + } + DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(4); + EvalQueryQuality result = dcg.evaluate("id", hits, rated); + assertEquals(12.392789260714371 , result.getQualityLevel(), 0.00001); + assertEquals(1, result.getUnknownDocs().size()); + + /** + * Check with normalization: to get the maximal possible dcg, sort documents by relevance in descending order + * + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * ------------------------------------------------------------------------------------------- + * 1 | 3 | 7.0 | 1.0  | 7.0 + * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 + * 3 | 2 | 3.0 | 2.0  | 1.5 + * 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339 + * ------------------------------------------------------------------------------------------- + * 5 | n.a | n.a | n.a.  | n.a. + * 6 | n.a | n.a | n.a  | n.a + * + * idcg = 13.347184833073591 (sum of last column) + */ + dcg.setNormalize(true); + assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); + } + public void testParseFromXContent() throws IOException { String xContent = " {\n" + " \"size\": 8,\n" diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java index 8924ff0f66d..181a3b11aac 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -19,10 +19,7 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -39,26 +36,23 @@ public class EvalQueryQualityTests extends ESTestCase { for (int i = 0; i < numberOfUnknownDocs; i++) { unknownDocs.add(RatedDocumentKeyTests.createRandomRatedDocumentKey()); } + int numberOfSearchHits = randomInt(5); + List ratedHits = new ArrayList<>(); + for (int i = 0; i < numberOfSearchHits; i++) { + ratedHits.add(RatedSearchHitTests.randomRatedSearchHit()); + } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAsciiOfLength(10), randomDoubleBetween(0.0, 1.0, true), unknownDocs); if (randomBoolean()) { // TODO randomize this evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(1, 5)); } + evalQueryQuality.addHitsAndRatings(ratedHits); return evalQueryQuality; } - private static EvalQueryQuality copy(EvalQueryQuality original) throws IOException { - try (BytesStreamOutput output = new BytesStreamOutput()) { - original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWritableRegistry)) { - return new EvalQueryQuality(in); - } - } - } - public void testSerialization() throws IOException { EvalQueryQuality original = randomEvalQueryQuality(); - EvalQueryQuality deserialized = copy(original); + EvalQueryQuality deserialized = RankEvalTestHelper.copy(original, EvalQueryQuality::new, namedWritableRegistry); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); @@ -67,13 +61,14 @@ public class EvalQueryQualityTests extends ESTestCase { public void testEqualsAndHash() throws IOException { EvalQueryQuality testItem = randomEvalQueryQuality(); RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - copy(testItem)); + RankEvalTestHelper.copy(testItem, EvalQueryQuality::new, namedWritableRegistry)); } private static EvalQueryQuality mutateTestItem(EvalQueryQuality original) { String id = original.getId(); double qualityLevel = original.getQualityLevel(); - List unknownDocs = original.getUnknownDocs(); + List unknownDocs = new ArrayList<>(original.getUnknownDocs()); + List ratedHits = new ArrayList<>(original.getHitsAndRatings()); MetricDetails breakdown = original.getMetricDetails(); switch (randomIntBetween(0, 3)) { case 0: @@ -83,7 +78,6 @@ public class EvalQueryQualityTests extends ESTestCase { qualityLevel = qualityLevel + 0.1; break; case 2: - unknownDocs = new ArrayList<>(unknownDocs); unknownDocs.add(RatedDocumentKeyTests.createRandomRatedDocumentKey()); break; case 3: @@ -93,13 +87,15 @@ public class EvalQueryQualityTests extends ESTestCase { breakdown = null; } break; + case 4: + ratedHits.add(RatedSearchHitTests.randomRatedSearchHit()); + break; default: - throw new IllegalStateException("The test should only allow three parameters mutated"); + throw new IllegalStateException("The test should only allow five parameters mutated"); } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(id, qualityLevel, unknownDocs); evalQueryQuality.addMetricDetails(breakdown); + evalQueryQuality.addHitsAndRatings(ratedHits); return evalQueryQuality; } - - } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index b6852be8311..a1495eaaf64 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.junit.Before; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -52,7 +53,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { ensureGreen(); client().prepareIndex("test", "testtype").setId("1") - .setSource("text", "berlin").get(); + .setSource("text", "berlin", "title", "Berlin, Germany").get(); client().prepareIndex("test", "testtype").setId("2") .setSource("text", "amsterdam").get(); client().prepareIndex("test", "testtype").setId("3") @@ -66,15 +67,19 @@ public class RankEvalRequestTests extends ESIntegTestCase { refresh(); } - public void testPrecisionAtRequest() { + public void testPrecisionAtRequest() throws IOException { List indices = Arrays.asList(new String[] { "test" }); List types = Arrays.asList(new String[] { "testtype" }); List specifications = new ArrayList<>(); SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - specifications.add(new RatedRequest("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5"))); - specifications.add(new RatedRequest("berlin_query", testQuery, indices, types, createRelevant("1"))); + RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5")); + amsterdamRequest.setSummaryFields(Arrays.asList(new String[]{ "text", "title" })); + specifications.add(amsterdamRequest); + RatedRequest berlinRequest = new RatedRequest("berlin_query", testQuery, indices, types, createRelevant("1")); + berlinRequest.setSummaryFields(Arrays.asList(new String[]{ "text", "title" })); + specifications.add(berlinRequest); RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtN(10)); @@ -86,11 +91,32 @@ public class RankEvalRequestTests extends ESIntegTestCase { Set> entrySet = response.getPartialResults().entrySet(); assertEquals(2, entrySet.size()); for (Entry entry : entrySet) { + EvalQueryQuality quality = entry.getValue(); if (entry.getKey() == "amsterdam_query") { - assertEquals(2, entry.getValue().getUnknownDocs().size()); + assertEquals(2, quality.getUnknownDocs().size()); + List hitsAndRatings = quality.getHitsAndRatings(); + assertEquals(6, hitsAndRatings.size()); + for (RatedSearchHit hit : hitsAndRatings) { + String id = hit.getSearchHit().getId(); + if (id.equals("1") || id.equals("6")) { + assertFalse(hit.getRating().isPresent()); + } else { + assertEquals(Rating.RELEVANT.ordinal(), hit.getRating().get().intValue()); + } + } } if (entry.getKey() == "berlin_query") { - assertEquals(5, entry.getValue().getUnknownDocs().size()); + assertEquals(5, quality.getUnknownDocs().size()); + List hitsAndRatings = quality.getHitsAndRatings(); + assertEquals(6, hitsAndRatings.size()); + for (RatedSearchHit hit : hitsAndRatings) { + String id = hit.getSearchHit().getId(); + if (id.equals("1")) { + assertEquals(Rating.RELEVANT.ordinal(), hit.getRating().get().intValue()); + } else { + assertFalse(hit.getRating().isPresent()); + } + } } } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java index 4218ae5d5cd..7c8733a9e89 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java @@ -20,6 +20,12 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -29,6 +35,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Collections; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; @@ -50,7 +57,7 @@ public class RankEvalTestHelper { return itemParser; } - public static void testHashCodeAndEquals(Object testItem, Object mutation, Object secondCopy) { + public static void testHashCodeAndEquals(T testItem, T mutation, T secondCopy) { assertFalse("testItem is equal to null", testItem.equals(null)); assertFalse("testItem is equal to incompatible type", testItem.equals("")); assertTrue("testItem is not equal to self", testItem.equals(testItem)); @@ -65,4 +72,31 @@ public class RankEvalTestHelper { assertThat("testItem copy's hashcode is different from original hashcode", secondCopy.hashCode(), equalTo(testItem.hashCode())); } + + /** + * Make a deep copy of an object by running it through a BytesStreamOutput + * @param original the original object + * @param reader a function able to create a new copy of this type + * @return a new copy of the original object + */ + public static T copy(T original, Writeable.Reader reader) throws IOException { + return copy(original, reader, new NamedWriteableRegistry(Collections.emptyList())); + } + + /** + * Make a deep copy of an object by running it through a BytesStreamOutput + * @param original the original object + * @param reader a function able to create a new copy of this type + * @param namedWriteableRegistry must be non-empty if the object itself or nested object implement {@link NamedWriteable} + * @return a new copy of the original object + */ + public static T copy(T original, Writeable.Reader reader, NamedWriteableRegistry namedWriteableRegistry) + throws IOException { + try (BytesStreamOutput output = new BytesStreamOutput()) { + original.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { + return reader.read(in); + } + } + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index c325181727e..01b749f759b 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -79,7 +79,15 @@ public class RatedRequestsTests extends ESTestCase { ratedDocs.add(RatedDocumentTests.createRatedDocument()); } - return new RatedRequest(specId, testRequest, indices, types, ratedDocs); + RatedRequest ratedRequest = new RatedRequest(specId, testRequest, indices, types, ratedDocs); + + List summaryFields = new ArrayList<>(); + int numSummaryFields = randomIntBetween(0, 5); + for (int i = 0; i < numSummaryFields; i++) { + summaryFields.add(randomAsciiOfLength(5)); + } + ratedRequest.setSummaryFields(summaryFields); + return ratedRequest; } public void testXContentRoundtrip() throws IOException { @@ -126,6 +134,7 @@ public class RatedRequestsTests extends ESTestCase { + " },\n" + " \"size\": 10\n" + " },\n" + + " \"summary_fields\" : [\"title\"],\n" + " \"ratings\": [ " + " {\"_index\": \"test\", \"_type\": \"testtype\", \"_id\": \"1\", \"rating\" : 1 }, " + " {\"_type\": \"testtype\", \"_index\": \"test\", \"_id\": \"2\", \"rating\" : 0 }, " diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java new file mode 100644 index 00000000000..2e65d1fe846 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.text.Text; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.Optional; + +public class RatedSearchHitTests extends ESTestCase { + + public static RatedSearchHit randomRatedSearchHit() { + Optional rating = randomBoolean() ? Optional.empty() : Optional.of(randomIntBetween(0, 5)); + SearchHit searchHit = new InternalSearchHit(randomIntBetween(0, 10), randomAsciiOfLength(10), new Text(randomAsciiOfLength(10)), + Collections.emptyMap()); + RatedSearchHit ratedSearchHit = new RatedSearchHit(searchHit, rating); + return ratedSearchHit; + } + + private static RatedSearchHit mutateTestItem(RatedSearchHit original) { + Optional rating = original.getRating(); + InternalSearchHit hit = (InternalSearchHit) original.getSearchHit(); + switch (randomIntBetween(0, 1)) { + case 0: + rating = rating.isPresent() ? Optional.of(rating.get() + 1) : Optional.of(randomInt(5)); + break; + case 1: + hit = new InternalSearchHit(hit.docId(), hit.getId() + randomAsciiOfLength(10), new Text(hit.getType()), + Collections.emptyMap()); + break; + default: + throw new IllegalStateException("The test should only allow two parameters mutated"); + } + return new RatedSearchHit(hit, rating); + } + + public void testSerialization() throws IOException { + RatedSearchHit original = randomRatedSearchHit(); + RatedSearchHit deserialized = RankEvalTestHelper.copy(original, RatedSearchHit::new); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + + public void testEqualsAndHash() throws IOException { + RatedSearchHit testItem = randomRatedSearchHit(); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), + RankEvalTestHelper.copy(testItem, RatedSearchHit::new)); + } +} diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 63944f899f1..36063a03a3f 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -59,13 +59,27 @@ "metric" : { "precisionatn": { "size": 10}} } - - match: {rank_eval.quality_level: 1} - - match: {rank_eval.details.amsterdam_query.quality_level: 1.0} - - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} - - match: {rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}} - - match: {rank_eval.details.berlin_query.quality_level: 1.0} - - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} - - match: {rank_eval.details.berlin_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}} + - match: { rank_eval.quality_level: 1} + - match: { rank_eval.details.amsterdam_query.quality_level: 1.0} + - match: { rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} + - match: { rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}} + + - length: { rank_eval.details.amsterdam_query.hits: 3} + - match: { rank_eval.details.amsterdam_query.hits.0.hit: {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "_score" : 0.44839138}} + - match: { rank_eval.details.amsterdam_query.hits.0.rating: 1} + - match: { rank_eval.details.amsterdam_query.hits.1.hit: {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "_score" : 0.44839138}} + - match: { rank_eval.details.amsterdam_query.hits.1.rating: 1} + - match: { rank_eval.details.amsterdam_query.hits.2.hit: {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "_score" : 0.21492207}} + - is_false: rank_eval.details.amsterdam_query.hits.2.rating + + - match: { rank_eval.details.berlin_query.quality_level: 1.0} + - match: { rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} + - match: { rank_eval.details.berlin_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}} + - length: { rank_eval.details.berlin_query.hits: 2} + - match: { rank_eval.details.berlin_query.hits.0.hit: { "_index" : "foo", "_type" : "bar", "_id" : "doc1", "_score" : 0.87138504}} + - match: { rank_eval.details.berlin_query.hits.0.rating: 1} + - match: { rank_eval.details.berlin_query.hits.1.hit: { "_index" : "foo", "_type" : "bar", "_id" : "doc4", "_score" : 0.41767058}} + - is_false: rank_eval.details.berlin_query.hits.1.rating --- "Reciprocal Rank": @@ -133,7 +147,7 @@ - match: {rank_eval.details.amsterdam_query.quality_level: 0.3333333333333333} - match: {rank_eval.details.amsterdam_query.metric_details: {"first_relevant": 3}} - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc2"}, - {"_index": "foo", "_type": "bar", "_id": "doc3"} ]} + {"_index": "foo", "_type": "bar", "_id": "doc3"} ]} - match: {rank_eval.details.berlin_query.quality_level: 0.5} - match: {rank_eval.details.berlin_query.metric_details: {"first_relevant": 2}} - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc1"}]} @@ -168,7 +182,7 @@ - match: {rank_eval.details.amsterdam_query.quality_level: 0} - match: {rank_eval.details.amsterdam_query.metric_details: {"first_relevant": -1}} - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc2"}, - {"_index": "foo", "_type": "bar", "_id": "doc3"} ]} + {"_index": "foo", "_type": "bar", "_id": "doc3"} ]} - match: {rank_eval.details.berlin_query.quality_level: 0.5} - match: {rank_eval.details.berlin_query.metric_details: {"first_relevant": 2}} - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc1"}]} From 9e394b0644a5fd6bab05bf8b965e58e002b09fa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 23 Sep 2016 14:45:48 +0200 Subject: [PATCH 060/297] Pull common operations into RankedListQualityMetric interface Currently each implementation of RankedListQualityMetric does some initial joining operation that links the input search hits with a rated document rating, if available. Also all metrics collect unknown docs and now also need to add the list of rated search hits to the partial query evaluation. This change centralizes this work in some new helper methods in RankedListQualityMetric. --- .../rankeval/DiscountedCumulativeGainAt.java | 50 +++++---------- ...RatedDocumentKey.java => DocumentKey.java} | 8 +-- .../index/rankeval/EvalQueryQuality.java | 16 +++-- .../index/rankeval/PrecisionAtN.java | 48 +++++--------- .../rankeval/RankedListQualityMetric.java | 46 ++++++++++--- .../index/rankeval/RatedDocument.java | 10 +-- .../index/rankeval/ReciprocalRank.java | 56 +++++----------- .../rankeval/TransportRankEvalAction.java | 18 +++--- ...entKeyTests.java => DocumentKeyTests.java} | 16 ++--- .../index/rankeval/EvalQueryQualityTests.java | 14 ++-- .../index/rankeval/PrecisionAtNTests.java | 64 ++++++++----------- .../index/rankeval/RankEvalResponseTests.java | 8 ++- .../index/rankeval/RankEvalTestHelper.java | 3 +- .../index/rankeval/ReciprocalRankTests.java | 42 +++++------- 14 files changed, 176 insertions(+), 223 deletions(-) rename modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/{RatedDocumentKey.java => DocumentKey.java} (91%) rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{RatedDocumentKeyTests.java => DocumentKeyTests.java} (77%) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index d4728f81fef..c998bb1e680 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -32,13 +32,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.Optional; +import java.util.stream.Collectors; -public class DiscountedCumulativeGainAt extends RankedListQualityMetric { +import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; +import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; + +public class DiscountedCumulativeGainAt implements RankedListQualityMetric { /** rank position up to which to check results. */ private int position; @@ -141,43 +142,24 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { @Override public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { - Map ratedDocsByKey = new HashMap<>(ratedDocs.size()); - List allRatings = new ArrayList<>(ratedDocs.size()); - for (RatedDocument doc : ratedDocs) { - ratedDocsByKey.put(doc.getKey(), doc); - allRatings.add(doc.getRating()); - } - - List unknownDocIds = new ArrayList<>(); - List hitsAndRatings = new ArrayList<>(); - List ratingsInSearchHits = new ArrayList<>(); - for (int i = 0; (i < position && i < hits.length); i++) { - RatedDocumentKey id = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); - RatedDocument ratedDoc = ratedDocsByKey.get(id); - if (ratedDoc != null) { - ratingsInSearchHits.add(ratedDoc.getRating()); - hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.of(ratedDoc.getRating()))); - } else { - unknownDocIds.add(id); - if (unknownDocRating != null) { - ratingsInSearchHits.add(unknownDocRating); - hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.of(unknownDocRating))); - } else { - // we add null here so that the later computation knows this position had no rating - ratingsInSearchHits.add(null); - hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.empty())); - } - } + List allRatings = ratedDocs.stream().mapToInt(RatedDocument::getRating).boxed().collect(Collectors.toList()); + List ratedHits = joinHitsWithRatings(hits, ratedDocs); + List ratingsInSearchHits = new ArrayList<>(Math.min(ratedHits.size(), position)); + for (RatedSearchHit hit : ratedHits.subList(0, position)) { + // unknownDocRating might be null, which means it will be unrated docs are ignored in the dcg calculation + // we still need to add them as a placeholder so the rank of the subsequent ratings is correct + ratingsInSearchHits.add(hit.getRating().orElse(unknownDocRating)); } double dcg = computeDCG(ratingsInSearchHits); if (normalize) { Collections.sort(allRatings, Comparator.nullsLast(Collections.reverseOrder())); - double idcg = computeDCG(allRatings.subList(0, Math.min(hits.length, allRatings.size()))); + double idcg = computeDCG(allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size()))); dcg = dcg / idcg; } - EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, dcg, unknownDocIds); - evalQueryQuality.addHitsAndRatings(hitsAndRatings); + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, dcg); + evalQueryQuality.addHitsAndRatings(ratedHits); + evalQueryQuality.setUnknownDocs(filterUnknownDocuments(ratedHits)); return evalQueryQuality; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java similarity index 91% rename from modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java rename to modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java index 35da907189f..55983d11e49 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java @@ -28,7 +28,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; import java.util.Objects; -public class RatedDocumentKey extends ToXContentToBytes implements Writeable { +public class DocumentKey extends ToXContentToBytes implements Writeable { private String docId; private String type; @@ -46,13 +46,13 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { this.docId = docId; } - public RatedDocumentKey(String index, String type, String docId) { + public DocumentKey(String index, String type, String docId) { this.index = index; this.type = type; this.docId = docId; } - public RatedDocumentKey(StreamInput in) throws IOException { + public DocumentKey(StreamInput in) throws IOException { this.index = in.readString(); this.type = in.readString(); this.docId = in.readString(); @@ -85,7 +85,7 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { if (obj == null || getClass() != obj.getClass()) { return false; } - RatedDocumentKey other = (RatedDocumentKey) obj; + DocumentKey other = (DocumentKey) obj; return Objects.equals(index, other.index) && Objects.equals(type, other.type) && Objects.equals(docId, other.docId); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index caa0ee7b179..edd220eadc2 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -39,20 +39,20 @@ import java.util.Objects;; public class EvalQueryQuality implements ToXContent, Writeable { /** documents seen as result for one request that were not annotated.*/ - private List unknownDocs; + private List unknownDocs = new ArrayList<>(); private String id; private double qualityLevel; private MetricDetails optionalMetricDetails; private List hits = new ArrayList<>(); - public EvalQueryQuality(String id, double qualityLevel, List unknownDocs) { + public EvalQueryQuality(String id, double qualityLevel) { this.id = id; - this.unknownDocs = unknownDocs; this.qualityLevel = qualityLevel; } public EvalQueryQuality(StreamInput in) throws IOException { - this(in.readString(), in.readDouble(), in.readList(RatedDocumentKey::new)); + this(in.readString(), in.readDouble()); + this.unknownDocs = in.readList(DocumentKey::new); this.hits = in.readList(RatedSearchHit::new); this.optionalMetricDetails = in.readOptionalNamedWriteable(MetricDetails.class); } @@ -74,7 +74,11 @@ public class EvalQueryQuality implements ToXContent, Writeable { return qualityLevel; } - public List getUnknownDocs() { + public void setUnknownDocs(List unknownDocs) { + this.unknownDocs = unknownDocs; + } + + public List getUnknownDocs() { return Collections.unmodifiableList(this.unknownDocs); } @@ -99,7 +103,7 @@ public class EvalQueryQuality implements ToXContent, Writeable { builder.startObject(id); builder.field("quality_level", this.qualityLevel); builder.startArray("unknown_docs"); - for (RatedDocumentKey key : unknownDocs) { + for (DocumentKey key : unknownDocs) { key.toXContent(builder, params); } builder.endArray(); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index ecf277037a0..ce28afdda68 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -29,22 +29,22 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import javax.naming.directory.SearchResult; +import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; +import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; + /** * Evaluate Precision at N, N being the number of search results to consider for precision calculation. * Documents of unkonwn quality are ignored in the precision at n computation and returned by document id. * By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the precision * calculation. This value can be changes using the "relevant_rating_threshold" parameter. * */ -public class PrecisionAtN extends RankedListQualityMetric { +public class PrecisionAtN implements RankedListQualityMetric { /** Number of results to check against a given set of relevant results. */ private int n; @@ -123,40 +123,24 @@ public class PrecisionAtN extends RankedListQualityMetric { **/ @Override public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { - - Map relevantDocIds = new HashMap<>(); - Map irrelevantDocIds = new HashMap<>(); - for (RatedDocument doc : ratedDocs) { - if (doc.getRating() >= this.relevantRatingThreshhold) { - relevantDocIds.put(doc.getKey(), doc); - } else { - irrelevantDocIds.put(doc.getKey(), doc); - } - } - int good = 0; int bad = 0; - List unknownDocIds = new ArrayList<>(); - List hitsAndRatings = new ArrayList<>(); - for (int i = 0; (i < n && i < hits.length); i++) { - RatedDocumentKey hitKey = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); - if (relevantDocIds.keySet().contains(hitKey)) { - RatedDocument ratedDocument = relevantDocIds.get(hitKey); - good++; - hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.of(ratedDocument.getRating()))); - } else if (irrelevantDocIds.keySet().contains(hitKey)) { - RatedDocument ratedDocument = irrelevantDocIds.get(hitKey); - bad++; - hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.of(ratedDocument.getRating()))); - } else { - unknownDocIds.add(hitKey); - hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.empty())); + List ratedSearchHits = joinHitsWithRatings(hits, ratedDocs); + for (RatedSearchHit hit : ratedSearchHits) { + Optional rating = hit.getRating(); + if (rating.isPresent()) { + if (rating.get() >= this.relevantRatingThreshhold) { + good++; + } else { + bad++; + } } } double precision = (double) good / (good + bad); - EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision, unknownDocIds); + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision); evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(good, good + bad)); - evalQueryQuality.addHitsAndRatings(hitsAndRatings); + evalQueryQuality.addHitsAndRatings(ratedSearchHits); + evalQueryQuality.setUnknownDocs(filterUnknownDocuments(ratedSearchHits)); return evalQueryQuality; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index a34cbd058cd..e42d88a49da 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -19,18 +19,21 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.NamedWriteable; -import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.SearchHit; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; /** * Classes implementing this interface provide a means to compute the quality of a result list @@ -38,20 +41,20 @@ import java.util.List; * * RelevancyLevel specifies the type of object determining the relevancy level of some known docid. * */ -public abstract class RankedListQualityMetric extends ToXContentToBytes implements NamedWriteable { +public interface RankedListQualityMetric extends ToXContent, NamedWriteable { /** * Returns a single metric representing the ranking quality of a set of returned documents * wrt. to a set of document Ids labeled as relevant for this search. * * @param taskId the id of the query for which the ranking is currently evaluated - * @param hits the result hits as returned by some search + * @param hits the result hits as returned by a search request * @param ratedDocs the documents that were ranked by human annotators for this query case * @return some metric representing the quality of the result hit list wrt. to relevant doc ids. * */ - public abstract EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs); + EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs); - public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { + static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { RankedListQualityMetric rc; Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { @@ -80,10 +83,33 @@ public abstract class RankedListQualityMetric extends ToXContentToBytes implemen return rc; } - double combine(Collection partialResults) { - return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size(); + static List joinHitsWithRatings(SearchHit[] hits, List ratedDocs) { + // join hits with rated documents + Map ratedDocumentMap = ratedDocs.stream() + .collect(Collectors.toMap(RatedDocument::getKey, item -> item)); + List ratedSearchHits = new ArrayList<>(hits.length); + for (SearchHit hit : hits) { + DocumentKey key = new DocumentKey(hit.getIndex(), hit.getType(), hit.getId()); + RatedDocument ratedDoc = ratedDocumentMap.get(key); + if (ratedDoc != null) { + ratedSearchHits.add(new RatedSearchHit(hit, Optional.of(ratedDoc.getRating()))); + } else { + ratedSearchHits.add(new RatedSearchHit(hit, Optional.empty())); + } + } + return ratedSearchHits; } - @Override - public abstract XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException; + static List filterUnknownDocuments(List ratedHits) { + // join hits with rated documents + List unknownDocs = ratedHits.stream() + .filter(hit -> hit.getRating().isPresent() == false) + .map(hit -> new DocumentKey(hit.getSearchHit().getIndex(), hit.getSearchHit().getType(), hit.getSearchHit().getId())) + .collect(Collectors.toList()); + return unknownDocs; + } + + default double combine(Collection partialResults) { + return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size(); + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index e0e608cc4ae..c6639619cac 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -54,23 +54,23 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { } private int rating; - private RatedDocumentKey key; + private DocumentKey key; public RatedDocument(String index, String type, String docId, int rating) { - this(new RatedDocumentKey(index, type, docId), rating); + this(new DocumentKey(index, type, docId), rating); } public RatedDocument(StreamInput in) throws IOException { - this.key = new RatedDocumentKey(in); + this.key = new DocumentKey(in); this.rating = in.readVInt(); } - public RatedDocument(RatedDocumentKey ratedDocumentKey, int rating) { + public RatedDocument(DocumentKey ratedDocumentKey, int rating) { this.key = ratedDocumentKey; this.rating = rating; } - public RatedDocumentKey getKey() { + public DocumentKey getKey() { return this.key; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index ce892b6ba6c..68ed594a013 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -29,23 +29,21 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import javax.naming.directory.SearchResult; +import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; +import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; + /** * Evaluate reciprocal rank. * By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the reciprocal rank * calculation. This value can be changes using the "relevant_rating_threshold" parameter. * */ -public class ReciprocalRank extends RankedListQualityMetric { +public class ReciprocalRank implements RankedListQualityMetric { public static final String NAME = "reciprocal_rank"; public static final int DEFAULT_MAX_ACCEPTABLE_RANK = 10; @@ -117,47 +115,25 @@ public class ReciprocalRank extends RankedListQualityMetric { **/ @Override public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { - Map ratedDocsByKey = new HashMap<>(); - for (RatedDocument doc : ratedDocs) { - ratedDocsByKey.put(doc.getKey(), doc); - } - - Set relevantDocIds = new HashSet<>(); - Set irrelevantDocIds = new HashSet<>(); - for (RatedDocument doc : ratedDocs) { - if (doc.getRating() >= this.relevantRatingThreshhold) { - relevantDocIds.add(doc.getKey()); - } else { - irrelevantDocIds.add(doc.getKey()); - } - } - - List unknownDocIds = new ArrayList<>(); - List hitsAndRatings = new ArrayList<>(); + List ratedHits = joinHitsWithRatings(hits, ratedDocs); int firstRelevant = -1; - boolean found = false; - for (int i = 0; i < hits.length; i++) { - RatedDocumentKey key = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); - RatedDocument ratedDocument = ratedDocsByKey.get(key); - if (ratedDocument != null) { - if (ratedDocument.getRating() >= this.relevantRatingThreshhold) { - if (found == false && i < maxAcceptableRank) { - firstRelevant = i + 1; // add one because rank is not - // 0-based - found = true; - } - hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.of(ratedDocument.getRating()))); + int rank = 1; + for (RatedSearchHit hit : ratedHits.subList(0, Math.min(maxAcceptableRank, ratedHits.size()))) { + Optional rating = hit.getRating(); + if (rating.isPresent()) { + if (rating.get() >= this.relevantRatingThreshhold) { + firstRelevant = rank; + break; } - } else { - unknownDocIds.add(key); - hitsAndRatings.add(new RatedSearchHit(hits[i], Optional.empty())); } + rank++; } double reciprocalRank = (firstRelevant == -1) ? 0 : 1.0d / firstRelevant; - EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, reciprocalRank, unknownDocIds); + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, reciprocalRank); evalQueryQuality.addMetricDetails(new Breakdown(firstRelevant)); - evalQueryQuality.addHitsAndRatings(hitsAndRatings); + evalQueryQuality.addHitsAndRatings(ratedHits); + evalQueryQuality.setUnknownDocs(filterUnknownDocuments(ratedHits)); return evalQueryQuality; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index ee5763da3c7..5225f12997e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -28,7 +28,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -64,7 +64,7 @@ public class TransportRankEvalAction extends HandledTransportAction listener) { RankEvalSpec qualityTask = request.getRankEvalSpec(); - Map> unknownDocs = new ConcurrentHashMap<>(); + Map> unknownDocs = new ConcurrentHashMap<>(); Collection specifications = qualityTask.getSpecifications(); AtomicInteger responseCounter = new AtomicInteger(specifications.size()); Map partialResults = new ConcurrentHashMap<>(specifications.size()); @@ -94,31 +94,31 @@ public class TransportRankEvalAction extends HandledTransportAction listener; private RatedRequest specification; - private Map partialResults; + private Map requestDetails; private RankEvalSpec task; private AtomicInteger responseCounter; public RankEvalActionListener(ActionListener listener, RankEvalSpec task, RatedRequest specification, - Map partialResults, Map> unknownDocs, + Map details, Map> unknownDocs, AtomicInteger responseCounter) { this.listener = listener; this.task = task; this.specification = specification; - this.partialResults = partialResults; + this.requestDetails = details; this.responseCounter = responseCounter; } @Override public void onResponse(SearchResponse searchResponse) { - SearchHits hits = searchResponse.getHits(); - EvalQueryQuality queryQuality = task.getEvaluator().evaluate(specification.getSpecId(), hits.getHits(), + SearchHit[] hits = searchResponse.getHits().getHits(); + EvalQueryQuality queryQuality = task.getEvaluator().evaluate(specification.getSpecId(), hits, specification.getRatedDocs()); - partialResults.put(specification.getSpecId(), queryQuality); + requestDetails.put(specification.getSpecId(), queryQuality); if (responseCounter.decrementAndGet() < 1) { // TODO add other statistics like micro/macro avg? listener.onResponse( - new RankEvalResponse(task.getEvaluator().combine(partialResults.values()), partialResults)); + new RankEvalResponse(task.getEvaluator().combine(requestDetails.values()), requestDetails)); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java similarity index 77% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java index 231a1b8e586..3cbdc4dc274 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java @@ -24,20 +24,20 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; -public class RatedDocumentKeyTests extends ESTestCase { +public class DocumentKeyTests extends ESTestCase { - static RatedDocumentKey createRandomRatedDocumentKey() { + static DocumentKey createRandomRatedDocumentKey() { String index = randomAsciiOfLengthBetween(0, 10); String type = randomAsciiOfLengthBetween(0, 10); String docId = randomAsciiOfLengthBetween(0, 10); - return new RatedDocumentKey(index, type, docId); + return new DocumentKey(index, type, docId); } - public RatedDocumentKey createRandomTestItem() { + public DocumentKey createRandomTestItem() { return createRandomRatedDocumentKey(); } - public RatedDocumentKey mutateTestItem(RatedDocumentKey original) { + public DocumentKey mutateTestItem(DocumentKey original) { String index = original.getIndex(); String type = original.getType(); String docId = original.getDocID(); @@ -54,12 +54,12 @@ public class RatedDocumentKeyTests extends ESTestCase { default: throw new IllegalStateException("The test should only allow three parameters mutated"); } - return new RatedDocumentKey(index, type, docId); + return new DocumentKey(index, type, docId); } public void testEqualsAndHash() throws IOException { - RatedDocumentKey testItem = createRandomRatedDocumentKey(); + DocumentKey testItem = createRandomRatedDocumentKey(); RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - new RatedDocumentKey(testItem.getIndex(), testItem.getType(), testItem.getDocID())); + new DocumentKey(testItem.getIndex(), testItem.getType(), testItem.getDocID())); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java index 181a3b11aac..ea465781ec6 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -31,21 +31,22 @@ public class EvalQueryQualityTests extends ESTestCase { private static NamedWriteableRegistry namedWritableRegistry = new NamedWriteableRegistry(new RankEvalPlugin().getNamedWriteables()); public static EvalQueryQuality randomEvalQueryQuality() { - List unknownDocs = new ArrayList<>(); + List unknownDocs = new ArrayList<>(); int numberOfUnknownDocs = randomInt(5); for (int i = 0; i < numberOfUnknownDocs; i++) { - unknownDocs.add(RatedDocumentKeyTests.createRandomRatedDocumentKey()); + unknownDocs.add(DocumentKeyTests.createRandomRatedDocumentKey()); } int numberOfSearchHits = randomInt(5); List ratedHits = new ArrayList<>(); for (int i = 0; i < numberOfSearchHits; i++) { ratedHits.add(RatedSearchHitTests.randomRatedSearchHit()); } - EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAsciiOfLength(10), randomDoubleBetween(0.0, 1.0, true), unknownDocs); + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAsciiOfLength(10), randomDoubleBetween(0.0, 1.0, true)); if (randomBoolean()) { // TODO randomize this evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(1, 5)); } + evalQueryQuality.setUnknownDocs(unknownDocs); evalQueryQuality.addHitsAndRatings(ratedHits); return evalQueryQuality; } @@ -67,7 +68,7 @@ public class EvalQueryQualityTests extends ESTestCase { private static EvalQueryQuality mutateTestItem(EvalQueryQuality original) { String id = original.getId(); double qualityLevel = original.getQualityLevel(); - List unknownDocs = new ArrayList<>(original.getUnknownDocs()); + List unknownDocs = new ArrayList<>(original.getUnknownDocs()); List ratedHits = new ArrayList<>(original.getHitsAndRatings()); MetricDetails breakdown = original.getMetricDetails(); switch (randomIntBetween(0, 3)) { @@ -78,7 +79,7 @@ public class EvalQueryQualityTests extends ESTestCase { qualityLevel = qualityLevel + 0.1; break; case 2: - unknownDocs.add(RatedDocumentKeyTests.createRandomRatedDocumentKey()); + unknownDocs.add(DocumentKeyTests.createRandomRatedDocumentKey()); break; case 3: if (breakdown == null) { @@ -93,7 +94,8 @@ public class EvalQueryQualityTests extends ESTestCase { default: throw new IllegalStateException("The test should only allow five parameters mutated"); } - EvalQueryQuality evalQueryQuality = new EvalQueryQuality(id, qualityLevel, unknownDocs); + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(id, qualityLevel); + evalQueryQuality.setUnknownDocs(unknownDocs); evalQueryQuality.addMetricDetails(breakdown); evalQueryQuality.addHitsAndRatings(ratedHits); return evalQueryQuality; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index e892126a37a..b65a860990e 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; @@ -36,17 +37,12 @@ import java.util.List; import java.util.Vector; import java.util.concurrent.ExecutionException; -import static java.util.Collections.emptyList; - public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - InternalSearchHit[] hits = new InternalSearchHit[1]; - hits[0] = new InternalSearchHit(0, "0", new Text("testtype"), Collections.emptyMap()); - hits[0].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); - EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated, "test", "testtype"), rated); assertEquals(1, evaluated.getQualityLevel(), 0.00001); assertEquals(1, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(1, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); @@ -59,12 +55,7 @@ public class PrecisionAtNTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); - InternalSearchHit[] hits = new InternalSearchHit[5]; - for (int i = 0; i < 5; i++) { - hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); - } - EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated, "test", "testtype"), rated); assertEquals((double) 4 / 5, evaluated.getQualityLevel(), 0.00001); assertEquals(4, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(5, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); @@ -81,14 +72,9 @@ public class PrecisionAtNTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "2", 2)); rated.add(new RatedDocument("test", "testtype", "3", 3)); rated.add(new RatedDocument("test", "testtype", "4", 4)); - InternalSearchHit[] hits = new InternalSearchHit[5]; - for (int i = 0; i < 5; i++) { - hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); - } PrecisionAtN precisionAtN = new PrecisionAtN(5); precisionAtN.setRelevantRatingThreshhold(2); - EvalQueryQuality evaluated = precisionAtN.evaluate("id", hits, rated); + EvalQueryQuality evaluated = precisionAtN.evaluate("id", toSearchHits(rated, "test", "testtype"), rated); assertEquals((double) 3 / 5, evaluated.getQualityLevel(), 0.00001); assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(5, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); @@ -98,15 +84,11 @@ public class PrecisionAtNTests extends ESTestCase { List rated = new ArrayList<>(); rated.add(new RatedDocument("test_other", "testtype", "0", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test_other", "testtype", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); - InternalSearchHit[] hits = new InternalSearchHit[5]; - for (int i = 0; i < 5; i++) { - hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); - } - EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "2", Rating.IRRELEVANT.ordinal())); + // the following search hits contain only the last three documents + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); assertEquals(2, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); @@ -116,15 +98,10 @@ public class PrecisionAtNTests extends ESTestCase { List rated = new ArrayList<>(); rated.add(new RatedDocument("test", "other_type", "0", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "other_type", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); - InternalSearchHit[] hits = new InternalSearchHit[5]; - for (int i = 0; i < 5; i++) { - hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); - } - EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "2", Rating.IRRELEVANT.ordinal())); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); assertEquals(2, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); @@ -144,9 +121,9 @@ public class PrecisionAtNTests extends ESTestCase { public void testCombine() { PrecisionAtN metric = new PrecisionAtN(); Vector partialResults = new Vector<>(3); - partialResults.add(new EvalQueryQuality("a", 0.1, emptyList())); - partialResults.add(new EvalQueryQuality("b", 0.2, emptyList())); - partialResults.add(new EvalQueryQuality("c", 0.6, emptyList())); + partialResults.add(new EvalQueryQuality("a", 0.1)); + partialResults.add(new EvalQueryQuality("b", 0.2)); + partialResults.add(new EvalQueryQuality("c", 0.6)); assertEquals(0.3, metric.combine(partialResults), Double.MIN_VALUE); } @@ -165,4 +142,13 @@ public class PrecisionAtNTests extends ESTestCase { assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); } + + private static SearchHit[] toSearchHits(List rated, String index, String type) { + InternalSearchHit[] hits = new InternalSearchHit[rated.size()]; + for (int i = 0; i < rated.size(); i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text(type), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0)); + } + return hits; + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index 73ba2301d58..fa79c43d3c6 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -41,11 +41,13 @@ public class RankEvalResponseTests extends ESTestCase { for (int i = 0; i < numberOfRequests; i++) { String id = randomAsciiOfLengthBetween(3, 10); int numberOfUnknownDocs = randomIntBetween(0, 5); - List unknownDocs = new ArrayList<>(numberOfUnknownDocs); + List unknownDocs = new ArrayList<>(numberOfUnknownDocs); for (int d = 0; d < numberOfUnknownDocs; d++) { - unknownDocs.add(RatedDocumentKeyTests.createRandomRatedDocumentKey()); + unknownDocs.add(DocumentKeyTests.createRandomRatedDocumentKey()); } - partials.put(id, new EvalQueryQuality(id, randomDoubleBetween(0.0, 1.0, true), unknownDocs)); + EvalQueryQuality evalQuality = new EvalQueryQuality(id, randomDoubleBetween(0.0, 1.0, true)); + evalQuality.setUnknownDocs(unknownDocs); + partials.put(id, evalQuality); } return new RankEvalResponse(randomDouble(), partials); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java index 7c8733a9e89..8be80707e06 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; @@ -46,7 +45,7 @@ import static org.junit.Assert.assertTrue; public class RankEvalTestHelper { - public static XContentParser roundtrip(ToXContentToBytes testItem) throws IOException { + public static XContentParser roundtrip(ToXContent testItem) throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(ESTestCase.randomFrom(XContentType.values())); if (ESTestCase.randomBoolean()) { builder.prettyPrint(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 6dc5e670db5..690b27337bf 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; @@ -35,8 +36,6 @@ import java.util.List; import java.util.Vector; import java.util.concurrent.ExecutionException; -import static java.util.Collections.emptyList; - public class ReciprocalRankTests extends ESTestCase { public void testMaxAcceptableRank() { @@ -47,11 +46,7 @@ public class ReciprocalRankTests extends ESTestCase { reciprocalRank.setMaxAcceptableRank(maxRank); assertEquals(maxRank, reciprocalRank.getMaxAcceptableRank()); - InternalSearchHit[] hits = new InternalSearchHit[10]; - for (int i = 0; i < 10; i++) { - hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); - } + SearchHit[] hits = toSearchHits(0, 9, "test", "type"); List ratedDocs = new ArrayList<>(); int relevantAt = 5; for (int i = 0; i < 10; i++) { @@ -81,11 +76,7 @@ public class ReciprocalRankTests extends ESTestCase { public void testEvaluationOneRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); - InternalSearchHit[] hits = new InternalSearchHit[10]; - for (int i = 0; i < 10; i++) { - hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); - } + SearchHit[] hits = toSearchHits(0, 9, "test", "type"); List ratedDocs = new ArrayList<>(); // mark one of the ten docs relevant int relevantAt = randomIntBetween(0, 9); @@ -114,11 +105,7 @@ public class ReciprocalRankTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "2", 2)); rated.add(new RatedDocument("test", "testtype", "3", 3)); rated.add(new RatedDocument("test", "testtype", "4", 4)); - InternalSearchHit[] hits = new InternalSearchHit[5]; - for (int i = 0; i < 5; i++) { - hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); - } + SearchHit[] hits = toSearchHits(0, 5, "test", "testtype"); ReciprocalRank reciprocalRank = new ReciprocalRank(); reciprocalRank.setRelevantRatingThreshhold(2); @@ -130,19 +117,15 @@ public class ReciprocalRankTests extends ESTestCase { public void testCombine() { ReciprocalRank reciprocalRank = new ReciprocalRank(); Vector partialResults = new Vector<>(3); - partialResults.add(new EvalQueryQuality("id1", 0.5, emptyList())); - partialResults.add(new EvalQueryQuality("id2", 1.0, emptyList())); - partialResults.add(new EvalQueryQuality("id3", 0.75, emptyList())); + partialResults.add(new EvalQueryQuality("id1", 0.5)); + partialResults.add(new EvalQueryQuality("id2", 1.0)); + partialResults.add(new EvalQueryQuality("id3", 0.75)); assertEquals(0.75, reciprocalRank.combine(partialResults), Double.MIN_VALUE); } public void testEvaluationNoRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); - InternalSearchHit[] hits = new InternalSearchHit[10]; - for (int i = 0; i < 10; i++) { - hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); - } + SearchHit[] hits = toSearchHits(0, 9, "test", "type"); List ratedDocs = new ArrayList<>(); EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); @@ -160,4 +143,13 @@ public class ReciprocalRankTests extends ESTestCase { assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); } + + private static SearchHit[] toSearchHits(int from, int to, String index, String type) { + InternalSearchHit[] hits = new InternalSearchHit[to - from]; + for (int i = from; i < to; i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text(type), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0)); + } + return hits; + } } From dfc6d1f3698c369780680ba92b95040a284e440b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 23 Sep 2016 16:20:27 +0200 Subject: [PATCH 061/297] Remove unknown docs from EvalQueryQuality The unknown document section in the response for each query can be rendered using the rated hits that are now also part of the response by just filtering the documents without a rating. --- .../rankeval/DiscountedCumulativeGainAt.java | 2 -- .../index/rankeval/EvalQueryQuality.java | 17 ++-------------- .../index/rankeval/PrecisionAtN.java | 2 -- .../index/rankeval/RatedSearchHit.java | 2 +- .../index/rankeval/ReciprocalRank.java | 2 -- .../DiscountedCumulativeGainAtTests.java | 11 ++++++---- .../index/rankeval/EvalQueryQualityTests.java | 20 +++++++------------ .../index/rankeval/RankEvalRequestTests.java | 7 +++++-- .../index/rankeval/RankEvalResponseTests.java | 1 - .../index/rankeval/ReciprocalRankTests.java | 18 ++++++++++------- 10 files changed, 33 insertions(+), 49 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index c998bb1e680..4f5718a702e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -36,7 +36,6 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; public class DiscountedCumulativeGainAt implements RankedListQualityMetric { @@ -159,7 +158,6 @@ public class DiscountedCumulativeGainAt implements RankedListQualityMetric { } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, dcg); evalQueryQuality.addHitsAndRatings(ratedHits); - evalQueryQuality.setUnknownDocs(filterUnknownDocuments(ratedHits)); return evalQueryQuality; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index edd220eadc2..6cd7cbac52a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects;; @@ -39,7 +38,6 @@ import java.util.Objects;; public class EvalQueryQuality implements ToXContent, Writeable { /** documents seen as result for one request that were not annotated.*/ - private List unknownDocs = new ArrayList<>(); private String id; private double qualityLevel; private MetricDetails optionalMetricDetails; @@ -52,7 +50,6 @@ public class EvalQueryQuality implements ToXContent, Writeable { public EvalQueryQuality(StreamInput in) throws IOException { this(in.readString(), in.readDouble()); - this.unknownDocs = in.readList(DocumentKey::new); this.hits = in.readList(RatedSearchHit::new); this.optionalMetricDetails = in.readOptionalNamedWriteable(MetricDetails.class); } @@ -61,7 +58,6 @@ public class EvalQueryQuality implements ToXContent, Writeable { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeDouble(qualityLevel); - out.writeList(unknownDocs); out.writeList(hits); out.writeOptionalNamedWriteable(this.optionalMetricDetails); } @@ -74,14 +70,6 @@ public class EvalQueryQuality implements ToXContent, Writeable { return qualityLevel; } - public void setUnknownDocs(List unknownDocs) { - this.unknownDocs = unknownDocs; - } - - public List getUnknownDocs() { - return Collections.unmodifiableList(this.unknownDocs); - } - public void addMetricDetails(MetricDetails breakdown) { this.optionalMetricDetails = breakdown; } @@ -103,7 +91,7 @@ public class EvalQueryQuality implements ToXContent, Writeable { builder.startObject(id); builder.field("quality_level", this.qualityLevel); builder.startArray("unknown_docs"); - for (DocumentKey key : unknownDocs) { + for (DocumentKey key : RankedListQualityMetric.filterUnknownDocuments(hits)) { key.toXContent(builder, params); } builder.endArray(); @@ -132,13 +120,12 @@ public class EvalQueryQuality implements ToXContent, Writeable { EvalQueryQuality other = (EvalQueryQuality) obj; return Objects.equals(id, other.id) && Objects.equals(qualityLevel, other.qualityLevel) && - Objects.equals(unknownDocs, other.unknownDocs) && Objects.equals(hits, other.hits) && Objects.equals(optionalMetricDetails, other.optionalMetricDetails); } @Override public final int hashCode() { - return Objects.hash(id, qualityLevel, unknownDocs, hits, optionalMetricDetails); + return Objects.hash(id, qualityLevel, hits, optionalMetricDetails); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index ce28afdda68..99d8b86a92f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -35,7 +35,6 @@ import java.util.Optional; import javax.naming.directory.SearchResult; -import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; /** @@ -140,7 +139,6 @@ public class PrecisionAtN implements RankedListQualityMetric { EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision); evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(good, good + bad)); evalQueryQuality.addHitsAndRatings(ratedSearchHits); - evalQueryQuality.setUnknownDocs(filterUnknownDocuments(ratedSearchHits)); return evalQueryQuality; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java index b2aaceeb34c..0378f273657 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java @@ -100,7 +100,7 @@ public class RatedSearchHit implements Writeable, ToXContent { @Override public final int hashCode() { - //NORELEASE for this to work requires InternalSearchHit to properly implement equals()/hashCode() + // NORELEASE for this to work requires InternalSearchHit to properly implement equals()/hashCode() XContentBuilder builder; String hitAsXContent; try { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 68ed594a013..caf52c0e17c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -35,7 +35,6 @@ import java.util.Optional; import javax.naming.directory.SearchResult; -import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; /** @@ -133,7 +132,6 @@ public class ReciprocalRank implements RankedListQualityMetric { EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, reciprocalRank); evalQueryQuality.addMetricDetails(new Breakdown(firstRelevant)); evalQueryQuality.addHitsAndRatings(ratedHits); - evalQueryQuality.setUnknownDocs(filterUnknownDocuments(ratedHits)); return evalQueryQuality; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java index 62f778c3204..9c5a5e3bb14 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java @@ -30,10 +30,13 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; +import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; + public class DiscountedCumulativeGainAtTests extends ESTestCase { /** @@ -110,7 +113,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(6); EvalQueryQuality result = dcg.evaluate("id", hits, rated); assertEquals(12.779642067948913, result.getQualityLevel(), 0.00001); - assertEquals(2, result.getUnknownDocs().size()); + assertEquals(2, filterUnknownDocuments(result.getHitsAndRatings()).size()); /** * Check with normalization: to get the maximal possible dcg, sort documents by relevance in descending order @@ -148,7 +151,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { */ public void testDCGAtFourMoreRatings() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1}; + Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1, null}; InternalSearchHit[] hits = new InternalSearchHit[6]; for (int i = 0; i < 6; i++) { if (i < relevanceRatings.length) { @@ -160,9 +163,9 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(4); - EvalQueryQuality result = dcg.evaluate("id", hits, rated); + EvalQueryQuality result = dcg.evaluate("id", Arrays.copyOfRange(hits, 0, 4), rated); assertEquals(12.392789260714371 , result.getQualityLevel(), 0.00001); - assertEquals(1, result.getUnknownDocs().size()); + assertEquals(1, filterUnknownDocuments(result.getHitsAndRatings()).size()); /** * Check with normalization: to get the maximal possible dcg, sort documents by relevance in descending order diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java index ea465781ec6..09e8c9a673a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -46,7 +46,6 @@ public class EvalQueryQualityTests extends ESTestCase { // TODO randomize this evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(1, 5)); } - evalQueryQuality.setUnknownDocs(unknownDocs); evalQueryQuality.addHitsAndRatings(ratedHits); return evalQueryQuality; } @@ -68,9 +67,8 @@ public class EvalQueryQualityTests extends ESTestCase { private static EvalQueryQuality mutateTestItem(EvalQueryQuality original) { String id = original.getId(); double qualityLevel = original.getQualityLevel(); - List unknownDocs = new ArrayList<>(original.getUnknownDocs()); List ratedHits = new ArrayList<>(original.getHitsAndRatings()); - MetricDetails breakdown = original.getMetricDetails(); + MetricDetails metricDetails = original.getMetricDetails(); switch (randomIntBetween(0, 3)) { case 0: id = id + "_"; @@ -79,24 +77,20 @@ public class EvalQueryQualityTests extends ESTestCase { qualityLevel = qualityLevel + 0.1; break; case 2: - unknownDocs.add(DocumentKeyTests.createRandomRatedDocumentKey()); - break; - case 3: - if (breakdown == null) { - breakdown = new PrecisionAtN.Breakdown(1, 5); + if (metricDetails == null) { + metricDetails = new PrecisionAtN.Breakdown(1, 5); } else { - breakdown = null; + metricDetails = null; } break; - case 4: + case 3: ratedHits.add(RatedSearchHitTests.randomRatedSearchHit()); break; default: - throw new IllegalStateException("The test should only allow five parameters mutated"); + throw new IllegalStateException("The test should only allow four parameters mutated"); } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(id, qualityLevel); - evalQueryQuality.setUnknownDocs(unknownDocs); - evalQueryQuality.addMetricDetails(breakdown); + evalQueryQuality.addMetricDetails(metricDetails); evalQueryQuality.addHitsAndRatings(ratedHits); return evalQueryQuality; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index a1495eaaf64..93d88fc3c72 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -36,6 +36,9 @@ import java.util.List; import java.util.Map.Entry; import java.util.Set; +import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; + + public class RankEvalRequestTests extends ESIntegTestCase { @Override protected Collection> transportClientPlugins() { @@ -93,7 +96,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { for (Entry entry : entrySet) { EvalQueryQuality quality = entry.getValue(); if (entry.getKey() == "amsterdam_query") { - assertEquals(2, quality.getUnknownDocs().size()); + assertEquals(2, filterUnknownDocuments(quality.getHitsAndRatings()).size()); List hitsAndRatings = quality.getHitsAndRatings(); assertEquals(6, hitsAndRatings.size()); for (RatedSearchHit hit : hitsAndRatings) { @@ -106,7 +109,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { } } if (entry.getKey() == "berlin_query") { - assertEquals(5, quality.getUnknownDocs().size()); + assertEquals(5, filterUnknownDocuments(quality.getHitsAndRatings()).size()); List hitsAndRatings = quality.getHitsAndRatings(); assertEquals(6, hitsAndRatings.size()); for (RatedSearchHit hit : hitsAndRatings) { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index fa79c43d3c6..ca524ad278b 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -46,7 +46,6 @@ public class RankEvalResponseTests extends ESTestCase { unknownDocs.add(DocumentKeyTests.createRandomRatedDocumentKey()); } EvalQueryQuality evalQuality = new EvalQueryQuality(id, randomDoubleBetween(0.0, 1.0, true)); - evalQuality.setUnknownDocs(unknownDocs); partials.put(id, evalQuality); } return new RankEvalResponse(randomDouble(), partials); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 690b27337bf..694d478ee41 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -46,7 +46,7 @@ public class ReciprocalRankTests extends ESTestCase { reciprocalRank.setMaxAcceptableRank(maxRank); assertEquals(maxRank, reciprocalRank.getMaxAcceptableRank()); - SearchHit[] hits = toSearchHits(0, 9, "test", "type"); + SearchHit[] hits = createSearchHits(0, 9, "test", "type"); List ratedDocs = new ArrayList<>(); int relevantAt = 5; for (int i = 0; i < 10; i++) { @@ -76,7 +76,7 @@ public class ReciprocalRankTests extends ESTestCase { public void testEvaluationOneRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); - SearchHit[] hits = toSearchHits(0, 9, "test", "type"); + SearchHit[] hits = createSearchHits(0, 9, "test", "type"); List ratedDocs = new ArrayList<>(); // mark one of the ten docs relevant int relevantAt = randomIntBetween(0, 9); @@ -105,7 +105,7 @@ public class ReciprocalRankTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "2", 2)); rated.add(new RatedDocument("test", "testtype", "3", 3)); rated.add(new RatedDocument("test", "testtype", "4", 4)); - SearchHit[] hits = toSearchHits(0, 5, "test", "testtype"); + SearchHit[] hits = createSearchHits(0, 5, "test", "testtype"); ReciprocalRank reciprocalRank = new ReciprocalRank(); reciprocalRank.setRelevantRatingThreshhold(2); @@ -125,7 +125,7 @@ public class ReciprocalRankTests extends ESTestCase { public void testEvaluationNoRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); - SearchHit[] hits = toSearchHits(0, 9, "test", "type"); + SearchHit[] hits = createSearchHits(0, 9, "test", "type"); List ratedDocs = new ArrayList<>(); EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); @@ -144,9 +144,13 @@ public class ReciprocalRankTests extends ESTestCase { assertEquals(testItem.hashCode(), parsedItem.hashCode()); } - private static SearchHit[] toSearchHits(int from, int to, String index, String type) { - InternalSearchHit[] hits = new InternalSearchHit[to - from]; - for (int i = from; i < to; i++) { + /** + * Create InternalSearchHits for testing, starting from dociId 'from' up to docId 'to'. + * The search hits index and type also need to be provided + */ + private static SearchHit[] createSearchHits(int from, int to, String index, String type) { + InternalSearchHit[] hits = new InternalSearchHit[to + 1 - from]; + for (int i = from; i <= to; i++) { hits[i] = new InternalSearchHit(i, i+"", new Text(type), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0)); } From 0b8a2e40cb309bc8782e61fc41e9f755a95098d3 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Tue, 1 Nov 2016 11:36:22 +0100 Subject: [PATCH 062/297] First step towards supporting templating in rank eval requests. (#20374) This adds support for templating in rank eval requests. Relates to #20231 Problem: In it's current state the rank-eval request API forces the user to repeat complete queries for each test request. In most use cases the structure of the query to test will be stable with only parameters changing across requests, so this looks like lots of boilerplate json for something that could be expressed in a more concise way. Uses templating/ ScriptServices to enable users to submit only one test request template and let them only specify template parameters on a per test request basis. --- .../java/org/elasticsearch/script/Script.java | 141 +++++++++--------- .../index/rankeval/PrecisionAtN.java | 2 +- .../index/rankeval/RankEvalContext.java | 14 +- .../index/rankeval/RankEvalSpec.java | 95 +++++++++--- .../index/rankeval/RatedRequest.java | 41 ++++- .../index/rankeval/RestRankEvalAction.java | 23 ++- .../rankeval/TransportRankEvalAction.java | 4 +- .../index/rankeval/RankEvalSpecTests.java | 48 +++++- .../index/rankeval/RatedRequestsTests.java | 16 +- .../test/rank_eval/10_basic.yaml | 2 +- .../build.gradle | 27 ++++ ...stRankEvalWithMustacheYAMLTestSuiteIT.java | 42 ++++++ .../test/rank-eval/30_template.yaml | 67 +++++++++ .../rest-api-spec/api/rank_eval.json | 15 +- .../rest-api-spec/api/rank_eval_template.json | 25 ++++ settings.gradle | 1 + 16 files changed, 454 insertions(+), 109 deletions(-) create mode 100644 qa/smoke-test-rank-eval-with-mustache/build.gradle create mode 100644 qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java create mode 100644 qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval_template.json diff --git a/core/src/main/java/org/elasticsearch/script/Script.java b/core/src/main/java/org/elasticsearch/script/Script.java index cfdbf966b05..da2f2ed9698 100644 --- a/core/src/main/java/org/elasticsearch/script/Script.java +++ b/core/src/main/java/org/elasticsearch/script/Script.java @@ -186,83 +186,84 @@ public final class Script implements ToXContent, Writeable { return builder; } - public static Script parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException { - return parse(parser, parseFieldMatcher, null); + public static Script parse(XContentParser parser, ParseFieldMatcher matcher) { + return parse(parser, matcher, null); } public static Script parse(XContentParser parser, QueryParseContext context) { + return parse(parser, context.getParseFieldMatcher(), null); + } + + public static Script parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher, @Nullable String lang) { try { - return parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); + XContentParser.Token token = parser.currentToken(); + // If the parser hasn't yet been pushed to the first token, do it now + if (token == null) { + token = parser.nextToken(); + } + if (token == XContentParser.Token.VALUE_STRING) { + return new Script(parser.text(), ScriptType.INLINE, lang, null); + } + if (token != XContentParser.Token.START_OBJECT) { + throw new ElasticsearchParseException("expected a string value or an object, but found [{}] instead", token); + } + String script = null; + ScriptType type = null; + Map params = null; + XContentType contentType = null; + String cfn = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + cfn = parser.currentName(); + } else if (parseFieldMatcher.match(cfn, ScriptType.INLINE.getParseField())) { + type = ScriptType.INLINE; + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + contentType = parser.contentType(); + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + script = builder.copyCurrentStructure(parser).bytes().utf8ToString(); + } else { + script = parser.text(); + } + } else if (parseFieldMatcher.match(cfn, ScriptType.FILE.getParseField())) { + type = ScriptType.FILE; + if (token == XContentParser.Token.VALUE_STRING) { + script = parser.text(); + } else { + throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); + } + } else if (parseFieldMatcher.match(cfn, ScriptType.STORED.getParseField())) { + type = ScriptType.STORED; + if (token == XContentParser.Token.VALUE_STRING) { + script = parser.text(); + } else { + throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); + } + } else if (parseFieldMatcher.match(cfn, ScriptField.LANG)) { + if (token == XContentParser.Token.VALUE_STRING) { + lang = parser.text(); + } else { + throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); + } + } else if (parseFieldMatcher.match(cfn, ScriptField.PARAMS)) { + if (token == XContentParser.Token.START_OBJECT) { + params = parser.map(); + } else { + throw new ElasticsearchParseException("expected an object for field [{}], but found [{}]", cfn, token); + } + } else { + throw new ElasticsearchParseException("unexpected field [{}]", cfn); + } + } + if (script == null) { + throw new ElasticsearchParseException("expected one of [{}], [{}] or [{}] fields, but found none", + ScriptType.INLINE.getParseField() .getPreferredName(), ScriptType.FILE.getParseField().getPreferredName(), + ScriptType.STORED.getParseField() .getPreferredName()); + } + return new Script(script, type, lang, params, contentType); } catch (IOException e) { throw new ParsingException(parser.getTokenLocation(), "Error parsing [" + ScriptField.SCRIPT.getPreferredName() + "] field", e); } - } - public static Script parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher, @Nullable String lang) throws IOException { - XContentParser.Token token = parser.currentToken(); - // If the parser hasn't yet been pushed to the first token, do it now - if (token == null) { - token = parser.nextToken(); - } - if (token == XContentParser.Token.VALUE_STRING) { - return new Script(parser.text(), ScriptType.INLINE, lang, null); - } - if (token != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("expected a string value or an object, but found [{}] instead", token); - } - String script = null; - ScriptType type = null; - Map params = null; - XContentType contentType = null; - String cfn = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - cfn = parser.currentName(); - } else if (parseFieldMatcher.match(cfn, ScriptType.INLINE.getParseField())) { - type = ScriptType.INLINE; - if (parser.currentToken() == XContentParser.Token.START_OBJECT) { - contentType = parser.contentType(); - XContentBuilder builder = XContentFactory.contentBuilder(contentType); - script = builder.copyCurrentStructure(parser).bytes().utf8ToString(); - } else { - script = parser.text(); - } - } else if (parseFieldMatcher.match(cfn, ScriptType.FILE.getParseField())) { - type = ScriptType.FILE; - if (token == XContentParser.Token.VALUE_STRING) { - script = parser.text(); - } else { - throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); - } - } else if (parseFieldMatcher.match(cfn, ScriptType.STORED.getParseField())) { - type = ScriptType.STORED; - if (token == XContentParser.Token.VALUE_STRING) { - script = parser.text(); - } else { - throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); - } - } else if (parseFieldMatcher.match(cfn, ScriptField.LANG)) { - if (token == XContentParser.Token.VALUE_STRING) { - lang = parser.text(); - } else { - throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); - } - } else if (parseFieldMatcher.match(cfn, ScriptField.PARAMS)) { - if (token == XContentParser.Token.START_OBJECT) { - params = parser.map(); - } else { - throw new ElasticsearchParseException("expected an object for field [{}], but found [{}]", cfn, token); - } - } else { - throw new ElasticsearchParseException("unexpected field [{}]", cfn); - } - } - if (script == null) { - throw new ElasticsearchParseException("expected one of [{}], [{}] or [{}] fields, but found none", - ScriptType.INLINE.getParseField() .getPreferredName(), ScriptType.FILE.getParseField().getPreferredName(), - ScriptType.STORED.getParseField() .getPreferredName()); - } - return new Script(script, type, lang, params, contentType); } @Override @@ -287,7 +288,7 @@ public final class Script implements ToXContent, Writeable { @Override public String toString() { return "[script: " + script + ", type: " + type.getParseField().getPreferredName() + ", lang: " - + lang + ", params: " + params + "]"; + + lang + ", params: " + params + ", contentType: " + contentType + "]"; } public interface ScriptField { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 99d8b86a92f..d47610c52ad 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -51,7 +51,7 @@ public class PrecisionAtN implements RankedListQualityMetric { /** ratings equal or above this value will be considered relevant. */ private int relevantRatingThreshhold = 1; - public static final String NAME = "precisionatn"; + public static final String NAME = "precision_atn"; private static final ParseField SIZE_FIELD = new ParseField("size"); private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java index 576fc594c9a..0e9a617bb59 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchExtRegistry; import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.aggregations.AggregatorParsers; @@ -33,11 +34,14 @@ public class RankEvalContext implements ParseFieldMatcherSupplier { private final SearchRequestParsers searchRequestParsers; private final ParseFieldMatcher parseFieldMatcher; private final QueryParseContext parseContext; + private final ScriptService scriptService; - public RankEvalContext(ParseFieldMatcher parseFieldMatcher, QueryParseContext parseContext, SearchRequestParsers searchRequestParsers) { + public RankEvalContext(ParseFieldMatcher parseFieldMatcher, QueryParseContext parseContext, SearchRequestParsers searchRequestParsers, + ScriptService scriptService) { this.parseFieldMatcher = parseFieldMatcher; this.searchRequestParsers = searchRequestParsers; this.parseContext = parseContext; + this.scriptService = scriptService; } public Suggesters getSuggesters() { @@ -48,6 +52,14 @@ public class RankEvalContext implements ParseFieldMatcherSupplier { return searchRequestParsers.aggParsers; } + public SearchRequestParsers getSearchRequestParsers() { + return searchRequestParsers; + } + + public ScriptService getScriptService() { + return scriptService; + } + public SearchExtRegistry getSearchExtParsers() { return searchRequestParsers.searchExtParsers; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index cadb88d77c0..4e7f28ac901 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -20,19 +20,27 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Objects; +import java.util.Map; /** * This class defines a ranking evaluation task including an id, a collection of queries to evaluate and the evaluation metric. @@ -42,11 +50,13 @@ import java.util.Objects; * */ public class RankEvalSpec extends ToXContentToBytes implements Writeable { - /** Collection of query specifications, that is e.g. search request templates to use for query translation. */ private Collection ratedRequests = new ArrayList<>(); /** Definition of the quality metric, e.g. precision at N */ private RankedListQualityMetric metric; + /** optional: Template to base test requests on */ + @Nullable + private Script template; public RankEvalSpec() { // TODO think if no args ctor is okay @@ -64,6 +74,9 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { ratedRequests.add(new RatedRequest(in)); } metric = in.readNamedWriteable(RankedListQualityMetric.class); + if (in.readBoolean()) { + template = new Script(in); + } } @Override @@ -73,22 +86,24 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { spec.writeTo(out); } out.writeNamedWriteable(metric); + if (template != null) { + out.writeBoolean(true); + template.writeTo(out); + } else { + out.writeBoolean(false); + } } - public void setEval(RankedListQualityMetric eval) { - this.metric = eval; + /** Set the metric to use for quality evaluation. */ + public void setMetric(RankedListQualityMetric metric) { + this.metric = metric; } - /** Returns the precision at n configuration (containing level of n to consider).*/ - public RankedListQualityMetric getEvaluator() { + /** Returns the metric to use for quality evaluation.*/ + public RankedListQualityMetric getMetric() { return metric; } - /** Sets the precision at n configuration (containing level of n to consider).*/ - public void setEvaluator(RankedListQualityMetric config) { - this.metric = config; - } - /** Returns a list of intent to query translation specifications to evaluate. */ public Collection getSpecifications() { return ratedRequests; @@ -98,19 +113,33 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { public void setSpecifications(Collection specifications) { this.ratedRequests = specifications; } + + /** Set the template to base test requests on. */ + public void setTemplate(Script script) { + this.template = script; + } + + /** Returns the template to base test requests on. */ + public Script getTemplate() { + return this.template; + } + private static final ParseField TEMPLATE_FIELD = new ParseField("template"); private static final ParseField METRIC_FIELD = new ParseField("metric"); private static final ParseField REQUESTS_FIELD = new ParseField("requests"); private static final ObjectParser PARSER = new ObjectParser<>("rank_eval", RankEvalSpec::new); static { - PARSER.declareObject(RankEvalSpec::setEvaluator, (p, c) -> { + PARSER.declareObject(RankEvalSpec::setMetric, (p, c) -> { try { return RankedListQualityMetric.fromXContent(p, c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } } , METRIC_FIELD); + PARSER.declareObject(RankEvalSpec::setTemplate, (p, c) -> { + return Script.parse(p, c.getParseFieldMatcher(), "mustache"); + }, TEMPLATE_FIELD); PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> { try { return RatedRequest.fromXContent(p, c); @@ -120,9 +149,43 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } , REQUESTS_FIELD); } + public static RankEvalSpec parse(XContentParser parser, RankEvalContext context, boolean templated) throws IOException { + RankEvalSpec spec = PARSER.parse(parser, context); + + if (templated) { + for (RatedRequest query_spec : spec.getSpecifications()) { + Map params = query_spec.getParams(); + Script scriptWithParams = new Script(spec.template.getScript(), spec.template.getType(), spec.template.getLang(), params); + String resolvedRequest = + ((BytesReference) + (context.getScriptService().executable(scriptWithParams, ScriptContext.Standard.SEARCH, params) + .run())) + .utf8ToString(); + try (XContentParser subParser = XContentFactory.xContent(resolvedRequest).createParser(resolvedRequest)) { + QueryParseContext parseContext = + new QueryParseContext( + context.getSearchRequestParsers().queryParsers, + subParser, + context.getParseFieldMatcher()); + SearchSourceBuilder templateResult = + SearchSourceBuilder.fromXContent( + parseContext, + context.getAggs(), + context.getSuggesters(), + context.getSearchExtParsers()); + query_spec.setTestRequest(templateResult); + } + } + } + return spec; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + if (this.template != null) { + builder.field(TEMPLATE_FIELD.getPreferredName(), this.template); + } builder.startArray(REQUESTS_FIELD.getPreferredName()); for (RatedRequest spec : this.ratedRequests) { spec.toXContent(builder, params); @@ -133,10 +196,6 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { return builder; } - public static RankEvalSpec parse(XContentParser parser, RankEvalContext context) throws IOException { - return PARSER.parse(parser, context); - } - @Override public final boolean equals(Object obj) { if (this == obj) { @@ -146,12 +205,14 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { return false; } RankEvalSpec other = (RankEvalSpec) obj; + return Objects.equals(ratedRequests, other.ratedRequests) && - Objects.equals(metric, other.metric); + Objects.equals(metric, other.metric) && + Objects.equals(template, other.template); } @Override public final int hashCode() { - return Objects.hash(ratedRequests, metric); + return Objects.hash(ratedRequests, metric, template); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index dbfc449b7ea..1cc3cc18a4a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -32,17 +32,21 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Objects; +import java.util.Map; +import java.util.Map.Entry; + /** * Defines a QA specification: All end user supplied query intents will be mapped to the search request specified in this search request * template and executed against the targetIndex given. Any filters that should be applied in the target system can be specified as well. * * The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery. * */ +@SuppressWarnings("unchecked") public class RatedRequest extends ToXContentToBytes implements Writeable { - private String specId; private SearchSourceBuilder testRequest; private List indices = new ArrayList<>(); @@ -50,6 +54,8 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { private List summaryFields = new ArrayList<>(); /** Collection of rated queries for this query QA specification.*/ private List ratedDocs = new ArrayList<>(); + /** Map of parameters to use for filling a query template, can be used instead of providing testRequest. */ + private Map params = new HashMap<>(); public RatedRequest() { // ctor that doesn't require all args to be present immediatly is easier to use with ObjectParser @@ -83,6 +89,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { for (int i = 0; i < intentSize; i++) { ratedDocs.add(new RatedDocument(in)); } + this.params = (Map) in.readMap(); int summaryFieldsSize = in.readInt(); summaryFields = new ArrayList<>(summaryFieldsSize); for (int i = 0; i < summaryFieldsSize; i++) { @@ -106,6 +113,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { for (RatedDocument ratedDoc : ratedDocs) { ratedDoc.writeTo(out); } + out.writeMap((Map) params); out.writeInt(summaryFields.size()); for (String fieldName : summaryFields) { out.writeString(fieldName); @@ -155,6 +163,14 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { public void setRatedDocs(List ratedDocs) { this.ratedDocs = ratedDocs; } + + public void setParams(Map params) { + this.params = params; + } + + public Map getParams() { + return this.params; + } public void setSummaryFields(List fields) { this.summaryFields = fields; @@ -168,6 +184,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { private static final ParseField ID_FIELD = new ParseField("id"); private static final ParseField REQUEST_FIELD = new ParseField("request"); private static final ParseField RATINGS_FIELD = new ParseField("ratings"); + private static final ParseField PARAMS_FIELD = new ParseField("params"); private static final ParseField FIELDS_FIELD = new ParseField("summary_fields"); private static final ObjectParser PARSER = new ObjectParser<>("requests", RatedRequest::new); @@ -187,7 +204,14 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex); } - } , RATINGS_FIELD); + }, RATINGS_FIELD); + PARSER.declareObject(RatedRequest::setParams, (p, c) -> { + try { + return (Map) p.map(); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex); + } + }, PARAMS_FIELD); } /** @@ -219,7 +243,13 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(ID_FIELD.getPreferredName(), this.specId); - builder.field(REQUEST_FIELD.getPreferredName(), this.testRequest); + if (testRequest != null) + builder.field(REQUEST_FIELD.getPreferredName(), this.testRequest); + builder.startObject(PARAMS_FIELD.getPreferredName()); + for (Entry entry : this.params.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); builder.startArray(RATINGS_FIELD.getPreferredName()); for (RatedDocument doc : this.ratedDocs) { doc.toXContent(builder, params); @@ -250,11 +280,12 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { Objects.equals(indices, other.indices) && Objects.equals(types, other.types) && Objects.equals(summaryFields, summaryFields) && - Objects.equals(ratedDocs, other.ratedDocs); + Objects.equals(ratedDocs, other.ratedDocs) && + Objects.equals(params, other.params); } @Override public final int hashCode() { - return Objects.hash(specId, testRequest, indices, types, summaryFields, ratedDocs); + return Objects.hash(specId, testRequest, indices, types, summaryFields, ratedDocs, params); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 3a629f94b45..ce06d794276 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchRequestParsers; import java.io.IOException; @@ -156,19 +157,31 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; * */ public class RestRankEvalAction extends BaseRestHandler { - private SearchRequestParsers searchRequestParsers; + private ScriptService scriptService; @Inject - public RestRankEvalAction(Settings settings, RestController controller, SearchRequestParsers searchRequestParsers) { + public RestRankEvalAction( + Settings settings, + RestController controller, + SearchRequestParsers searchRequestParsers, + ScriptService scriptService) { super(settings); this.searchRequestParsers = searchRequestParsers; + this.scriptService = scriptService; controller.registerHandler(GET, "/_rank_eval", this); controller.registerHandler(POST, "/_rank_eval", this); controller.registerHandler(GET, "/{index}/_rank_eval", this); controller.registerHandler(POST, "/{index}/_rank_eval", this); controller.registerHandler(GET, "/{index}/{type}/_rank_eval", this); controller.registerHandler(POST, "/{index}/{type}/_rank_eval", this); + + controller.registerHandler(GET, "/_rank_eval/template", this); + controller.registerHandler(POST, "/_rank_eval/template", this); + controller.registerHandler(GET, "/{index}/_rank_eval/template", this); + controller.registerHandler(POST, "/{index}/_rank_eval/template", this); + controller.registerHandler(GET, "/{index}/{type}/_rank_eval/template", this); + controller.registerHandler(POST, "/{index}/{type}/_rank_eval/template", this); } @Override @@ -180,7 +193,7 @@ public class RestRankEvalAction extends BaseRestHandler { if (restContent != null) { parseRankEvalRequest(rankEvalRequest, request, // TODO can we get rid of aggregators parsers and suggesters? - new RankEvalContext(parseFieldMatcher, parseContext, searchRequestParsers)); + new RankEvalContext(parseFieldMatcher, parseContext, searchRequestParsers, scriptService)); } } return channel -> client.executeLocally(RankEvalAction.INSTANCE, rankEvalRequest, @@ -191,7 +204,9 @@ public class RestRankEvalAction extends BaseRestHandler { throws IOException { List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); List types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type"))); - RankEvalSpec spec = RankEvalSpec.parse(context.parser(), context); + RankEvalSpec spec = null; + boolean containsTemplate = request.path().contains("template"); + spec = RankEvalSpec.parse(context.parser(), context, containsTemplate); for (RatedRequest specification : spec.getSpecifications()) { specification.setIndices(indices); specification.setTypes(types); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 5225f12997e..d632534776f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -111,14 +111,14 @@ public class TransportRankEvalAction extends HandledTransportAction params = randomBoolean() ? null : Collections.singletonMap("key", "value"); + ScriptService.ScriptType scriptType = randomFrom(ScriptService.ScriptType.values()); + String script; + if (scriptType == ScriptService.ScriptType.INLINE) { + try (XContentBuilder builder = XContentBuilder.builder(xContent)) { + builder.startObject(); + builder.field("field", randomAsciiOfLengthBetween(1, 5)); + builder.endObject(); + script = builder.string(); + } + } else { + script = randomAsciiOfLengthBetween(1, 5); + } + + testItem.setTemplate(new Script( + script, + scriptType, + randomFrom("_lang1", "_lang2", null), + params, + scriptType == ScriptService.ScriptType.INLINE ? xContent.type() : null)); + } + + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + if (ESTestCase.randomBoolean()) { + builder.prettyPrint(); + } + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); + XContentBuilder shuffled = ESTestCase.shuffleXContent(builder); + XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, - searchRequestParsers); + searchRequestParsers, null); - RankEvalSpec parsedItem = RankEvalSpec.parse(itemParser, rankContext); + RankEvalSpec parsedItem = RankEvalSpec.parse(itemParser, rankContext, false); // IRL these come from URL parameters - see RestRankEvalAction parsedItem.getSpecifications().stream().forEach(e -> {e.setIndices(indices); e.setTypes(types);}); assertNotSame(testItem, parsedItem); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 01b749f759b..51d08b21c50 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -38,7 +38,9 @@ import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static java.util.Collections.emptyList; @@ -81,6 +83,16 @@ public class RatedRequestsTests extends ESTestCase { RatedRequest ratedRequest = new RatedRequest(specId, testRequest, indices, types, ratedDocs); + + if (randomBoolean()) { + Map params = new HashMap(); + int randomSize = randomIntBetween(1, 10); + for (int i = 0; i < randomSize; i++) { + params.put(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); + } + ratedRequest.setParams(params); + } + List summaryFields = new ArrayList<>(); int numSummaryFields = randomIntBetween(0, 5); for (int i = 0; i < numSummaryFields; i++) { @@ -109,7 +121,7 @@ public class RatedRequestsTests extends ESTestCase { QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, - searchRequestParsers); + searchRequestParsers, null); RatedRequest parsedItem = RatedRequest.fromXContent(itemParser, rankContext); parsedItem.setIndices(indices); // IRL these come from URL parameters - see RestRankEvalAction @@ -143,7 +155,7 @@ public class RatedRequestsTests extends ESTestCase { XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, - searchRequestParsers); + searchRequestParsers, null); RatedRequest specification = RatedRequest.fromXContent(parser, rankContext); assertEquals("my_qa_query", specification.getSpecId()); assertNotNull(specification.getTestRequest()); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 36063a03a3f..da6646cb0a5 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -56,7 +56,7 @@ "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] } ], - "metric" : { "precisionatn": { "size": 10}} + "metric" : { "precision_atn": { "size": 10}} } - match: { rank_eval.quality_level: 1} diff --git a/qa/smoke-test-rank-eval-with-mustache/build.gradle b/qa/smoke-test-rank-eval-with-mustache/build.gradle new file mode 100644 index 00000000000..4860d5469af --- /dev/null +++ b/qa/smoke-test-rank-eval-with-mustache/build.gradle @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.rest-test' + +/* +dependencies { + testCompile project(path: ':modules:rank-eval', configuration: 'runtime') + testCompile project(path: ':modules:lang-mustache', configuration: 'runtime') +} +*/ diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java new file mode 100644 index 00000000000..6fc2829c4a4 --- /dev/null +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.smoketest; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; + +import java.io.IOException; + +public class SmokeTestRankEvalWithMustacheYAMLTestSuiteIT extends ESClientYamlSuiteTestCase { + + public SmokeTestRankEvalWithMustacheYAMLTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, ClientYamlTestParseException { + return ESClientYamlSuiteTestCase.createParameters(0, 1); + } + +} diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml new file mode 100644 index 00000000000..bf94ea6ae8f --- /dev/null +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml @@ -0,0 +1,67 @@ +--- +"Template request": + - do: + indices.create: + index: foo + body: + settings: + index: + number_of_shards: 1 + - do: + index: + index: foo + type: bar + id: doc1 + body: { "text": "berlin" } + + - do: + index: + index: foo + type: bar + id: doc2 + body: { "text": "amsterdam" } + + - do: + index: + index: foo + type: bar + id: doc3 + body: { "text": "amsterdam" } + + - do: + index: + index: foo + type: bar + id: doc4 + body: { "text": "something about amsterdam and berlin" } + + - do: + indices.refresh: {} + + - do: + rank_eval_template: + body: { + "template": { + "inline": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" + }, + "requests" : [ + { + "id": "amsterdam_query", + "params": { "query_string": "amsterdam" }, + "ratings": [ + {"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 0}, + {"_index": "foo", "_type": "bar", "_id": "doc2", "rating": 1}, + {"_index": "foo", "_type": "bar", "_id": "doc3", "rating": 1}] + }, + { + "id" : "berlin_query", + "params": { "query_string": "berlin" }, + "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] + } + ], + "metric" : { "precision_atn": { "size": 10}} + } + + - match: {rank_eval.quality_level: 1} + - match: {rank_eval.details.berlin_query.unknown_docs.0._id: "doc4"} + - match: {rank_eval.details.amsterdam_query.unknown_docs.0._id: "doc4"} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json index 681eb9f6081..37ff11f8764 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json @@ -4,9 +4,18 @@ "methods": ["POST"], "url": { "path": "/_rank_eval", - "paths": ["/_rank_eval"], - "parts": {}, - "params": {} + "paths": ["/_rank_eval", "/{index}/_rank_eval", "/{index}/{type}/_rank_eval"], + "parts": { + "index": { + "type": "list", + "description" : "A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices" + }, + "type": { + "type" : "list", + "description" : "A comma-separated list of document types to search; leave empty to perform the operation on all types" + } + }, + "params": {} }, "body": { "description": "The search definition using the Query DSL and the prototype for the eval request.", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval_template.json new file mode 100644 index 00000000000..abb21c273e4 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval_template.json @@ -0,0 +1,25 @@ +{ + "rank_eval_template": { + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-rank-eval.html", + "methods": ["POST"], + "url": { + "path": "/_rank_eval/template", + "paths": ["/_rank_eval/template", "/{index}/_rank_eval/template", "/{index}/{type}/_rank_eval/template"], + "parts": { + "index": { + "type": "list", + "description" : "A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices" + }, + "type": { + "type" : "list", + "description" : "A comma-separated list of document types to search; leave empty to perform the operation on all types" + } + }, + "params": {} + }, + "body": { + "description": "The search definition using the Query DSL and the prototype for the eval request.", + "required": true + } + } +} diff --git a/settings.gradle b/settings.gradle index aee42e1e923..4f249de4391 100644 --- a/settings.gradle +++ b/settings.gradle @@ -61,6 +61,7 @@ List projects = [ 'qa:smoke-test-ingest-with-all-dependencies', 'qa:smoke-test-ingest-disabled', 'qa:smoke-test-multinode', + 'qa:smoke-test-rank-eval-with-mustache', 'qa:smoke-test-plugins', 'qa:smoke-test-reindex-with-painless', 'qa:smoke-test-http', From 51102ee91cd8d2b66709ef3942129c38e11fdcd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 1 Nov 2016 14:42:24 +0100 Subject: [PATCH 063/297] Fixing compile issue with ScriptType after merge with master --- .../index/rankeval/RankEvalSpecTests.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 0f26d7ab61f..a2e74886f64 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -32,7 +32,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.aggregations.AggregatorParsers; @@ -104,9 +104,9 @@ public class RankEvalSpecTests extends ESTestCase { if (randomBoolean()) { final Map params = randomBoolean() ? null : Collections.singletonMap("key", "value"); - ScriptService.ScriptType scriptType = randomFrom(ScriptService.ScriptType.values()); + ScriptType scriptType = randomFrom(ScriptType.values()); String script; - if (scriptType == ScriptService.ScriptType.INLINE) { + if (scriptType == ScriptType.INLINE) { try (XContentBuilder builder = XContentBuilder.builder(xContent)) { builder.startObject(); builder.field("field", randomAsciiOfLengthBetween(1, 5)); @@ -116,13 +116,13 @@ public class RankEvalSpecTests extends ESTestCase { } else { script = randomAsciiOfLengthBetween(1, 5); } - + testItem.setTemplate(new Script( script, scriptType, randomFrom("_lang1", "_lang2", null), params, - scriptType == ScriptService.ScriptType.INLINE ? xContent.type() : null)); + scriptType == ScriptType.INLINE ? xContent.type() : null)); } XContentBuilder builder = XContentFactory.contentBuilder(contentType); From 25565b9baae88b67398f4ae4455266fda9bdeeac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 27 Oct 2016 12:22:32 +0200 Subject: [PATCH 064/297] RankEval: Check for duplicate keys in rated documents When multiple ratings for the same document (identified by _index, _type, _id) are specified in the request we should throw an error. This change adds a check for this in the RatedRequest setter (and ctor that uses that setter). Closes #20997 --- .../index/rankeval/RatedRequest.java | 17 +++++++++++++++-- .../index/rankeval/RatedRequestsTests.java | 11 +++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 1cc3cc18a4a..4ac76877816 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -33,8 +33,10 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.Map; import java.util.Map.Entry; @@ -68,7 +70,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { this.testRequest = testRequest; this.indices = indices; this.types = types; - this.ratedDocs = ratedDocs; + setRatedDocs(ratedDocs); } public RatedRequest(StreamInput in) throws IOException { @@ -159,8 +161,19 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { return ratedDocs; } - /** Set a list of rated documents for this query. */ + /** + * Set a list of rated documents for this query. + * No documents with same _index/_type/id allowed. + **/ public void setRatedDocs(List ratedDocs) { + Set docKeys = new HashSet<>(); + for (RatedDocument doc : ratedDocs) { + if (docKeys.add(doc.getKey()) == false) { + String docKeyToString = doc.getKey().toString().replaceAll("\n", "").replaceAll(" ", " "); + throw new IllegalArgumentException( + "Found duplicate rated document key [" + docKeyToString + "]"); + } + } this.ratedDocs = ratedDocs; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 51d08b21c50..19471c20d09 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -39,6 +39,7 @@ import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -131,6 +132,16 @@ public class RatedRequestsTests extends ESTestCase { assertEquals(testItem.hashCode(), parsedItem.hashCode()); } + public void testDuplicateRatedDocThrowsException() { + RatedRequest request = createTestItem(Arrays.asList("index"), Arrays.asList("type")); + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1), + new RatedDocument(new DocumentKey("index1", "type1", "id1"), 5)); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> request.setRatedDocs(ratedDocs)); + assertEquals( + "Found duplicate rated document key [{ \"_index\" : \"index1\", \"_type\" : \"type1\", \"_id\" : \"id1\"}]", + ex.getMessage()); + } + public void testParseFromXContent() throws IOException { // we modify the order of index/type/docId to make sure it doesn't matter for parsing xContent String querySpecString = " {\n" From 2dad72e68c540342a98be9cfa3a6f335b44ba6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 3 Nov 2016 12:30:39 +0100 Subject: [PATCH 065/297] Rank Eval: Handle precion@ edge case There's a currently unhandled edge case for the precion@ metric. When none of the search hits in the result are rated, we have neither true nor false positives which currently leads to division by zero. We should return a precion of 0.0 in this case. --- .../index/rankeval/PrecisionAtN.java | 15 +++++++++------ .../index/rankeval/PrecisionAtNTests.java | 19 +++++++++++++------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index d47610c52ad..c639e9fc376 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -122,22 +122,25 @@ public class PrecisionAtN implements RankedListQualityMetric { **/ @Override public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { - int good = 0; - int bad = 0; + int truePositives = 0; + int falsePositives = 0; List ratedSearchHits = joinHitsWithRatings(hits, ratedDocs); for (RatedSearchHit hit : ratedSearchHits) { Optional rating = hit.getRating(); if (rating.isPresent()) { if (rating.get() >= this.relevantRatingThreshhold) { - good++; + truePositives++; } else { - bad++; + falsePositives++; } } } - double precision = (double) good / (good + bad); + double precision = 0.0; + if (truePositives + falsePositives > 0) { + precision = (double) truePositives / (truePositives + falsePositives); + } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision); - evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(good, good + bad)); + evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(truePositives, truePositives + falsePositives)); evalQueryQuality.addHitsAndRatings(ratedSearchHits); return evalQueryQuality; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index b65a860990e..800a38c0ec9 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -35,11 +35,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Vector; -import java.util.concurrent.ExecutionException; public class PrecisionAtNTests extends ESTestCase { - public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { + public void testPrecisionAtFiveCalculation() { List rated = new ArrayList<>(); rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated, "test", "testtype"), rated); @@ -48,7 +47,7 @@ public class PrecisionAtNTests extends ESTestCase { assertEquals(1, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } - public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException { + public void testPrecisionAtFiveIgnoreOneResult() { List rated = new ArrayList<>(); rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); @@ -65,7 +64,7 @@ public class PrecisionAtNTests extends ESTestCase { * test that the relevant rating threshold can be set to something larger than 1. * e.g. we set it to 2 here and expect dics 0-2 to be not relevant, doc 3 and 4 to be relevant */ - public void testPrecisionAtFiveRelevanceThreshold() throws IOException, InterruptedException, ExecutionException { + public void testPrecisionAtFiveRelevanceThreshold() { List rated = new ArrayList<>(); rated.add(new RatedDocument("test", "testtype", "0", 0)); rated.add(new RatedDocument("test", "testtype", "1", 1)); @@ -80,7 +79,7 @@ public class PrecisionAtNTests extends ESTestCase { assertEquals(5, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } - public void testPrecisionAtFiveCorrectIndex() throws IOException, InterruptedException, ExecutionException { + public void testPrecisionAtFiveCorrectIndex() { List rated = new ArrayList<>(); rated.add(new RatedDocument("test_other", "testtype", "0", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test_other", "testtype", "1", Rating.RELEVANT.ordinal())); @@ -94,7 +93,7 @@ public class PrecisionAtNTests extends ESTestCase { assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } - public void testPrecisionAtFiveCorrectType() throws IOException, InterruptedException, ExecutionException { + public void testPrecisionAtFiveCorrectType() { List rated = new ArrayList<>(); rated.add(new RatedDocument("test", "other_type", "0", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "other_type", "1", Rating.RELEVANT.ordinal())); @@ -107,6 +106,14 @@ public class PrecisionAtNTests extends ESTestCase { assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } + public void testNoRatedDocs() throws Exception { + List rated = new ArrayList<>(); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated, "test", "testtype"), rated); + assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); + assertEquals(0, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(0, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + } + public void testParseFromXContent() throws IOException { String xContent = " {\n" + " \"size\": 10,\n" From d013a2c8f12ce53bba58ecfff824efab6a9d8972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 8 Nov 2016 11:30:36 +0100 Subject: [PATCH 066/297] Adapting to change in ESClientYamlSuiteTestCase api --- .../java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java index 4d01d183fc7..09e1a906044 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java @@ -35,6 +35,6 @@ public class RankEvalYamlIT extends ESClientYamlSuiteTestCase { @ParametersFactory public static Iterable parameters() throws IOException, ClientYamlTestParseException { - return ESClientYamlSuiteTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(); } } From 3855d7f721c66afa02d74495ca2c0ebbc60bf397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 3 Nov 2016 16:45:42 +0100 Subject: [PATCH 067/297] Removing the 'size' parameter from precision metric --- .../{PrecisionAtN.java => Precision.java} | 73 +++++++------------ .../index/rankeval/RankEvalPlugin.java | 6 +- .../rankeval/RankedListQualityMetric.java | 4 +- .../index/rankeval/EvalQueryQualityTests.java | 4 +- ...isionAtNTests.java => PrecisionTests.java} | 65 +++++++++-------- .../index/rankeval/RankEvalRequestTests.java | 6 +- .../index/rankeval/RankEvalSpecTests.java | 19 ++--- .../index/rankeval/ReciprocalRankTests.java | 2 +- .../test/rank_eval/10_basic.yaml | 2 +- 9 files changed, 77 insertions(+), 104 deletions(-) rename modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/{PrecisionAtN.java => Precision.java} (75%) rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{PrecisionAtNTests.java => PrecisionTests.java} (69%) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java similarity index 75% rename from modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java rename to modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java index c639e9fc376..4871a1e5359 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java @@ -23,7 +23,7 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; @@ -38,38 +38,36 @@ import javax.naming.directory.SearchResult; import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; /** - * Evaluate Precision at N, N being the number of search results to consider for precision calculation. - * Documents of unkonwn quality are ignored in the precision at n computation and returned by document id. - * By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the precision - * calculation. This value can be changes using the "relevant_rating_threshold" parameter. - * */ -public class PrecisionAtN implements RankedListQualityMetric { - - /** Number of results to check against a given set of relevant results. */ - private int n; + * Evaluate Precision of the search results. Documents without a rating are + * ignored. By default documents with a rating equal or bigger than 1 are + * considered to be "relevant" for the precision calculation. This value can be + * changes using the "relevant_rating_threshold" parameter. + */ +public class Precision implements RankedListQualityMetric { /** ratings equal or above this value will be considered relevant. */ private int relevantRatingThreshhold = 1; - public static final String NAME = "precision_atn"; + public static final String NAME = "precision"; - private static final ParseField SIZE_FIELD = new ParseField("size"); private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "precision_at", a -> new PrecisionAtN((Integer) a[0])); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Precision::new); - static { - PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD); - PARSER.declareInt(PrecisionAtN::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); + public Precision() { + // needed for supplier in parser } - public PrecisionAtN(StreamInput in) throws IOException { - n = in.readInt(); + static { + PARSER.declareInt(Precision::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); + } + + public Precision(StreamInput in) throws IOException { + relevantRatingThreshhold = in.readOptionalVInt(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeInt(n); + out.writeOptionalVInt(relevantRatingThreshhold); } @Override @@ -77,27 +75,6 @@ public class PrecisionAtN implements RankedListQualityMetric { return NAME; } - /** - * Initialises n with 10 - * */ - public PrecisionAtN() { - this.n = 10; - } - - /** - * @param n number of top results to check against a given set of relevant results. - * */ - public PrecisionAtN(int n) { - this.n= n; - } - - /** - * Return number of search results to check for quality. - * */ - public int getN() { - return n; - } - /** * Sets the rating threshold above which ratings are considered to be "relevant" for this metric. * */ @@ -113,7 +90,7 @@ public class PrecisionAtN implements RankedListQualityMetric { return relevantRatingThreshhold ; } - public static PrecisionAtN fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { + public static Precision fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } @@ -140,7 +117,7 @@ public class PrecisionAtN implements RankedListQualityMetric { precision = (double) truePositives / (truePositives + falsePositives); } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision); - evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(truePositives, truePositives + falsePositives)); + evalQueryQuality.addMetricDetails(new Precision.Breakdown(truePositives, truePositives + falsePositives)); evalQueryQuality.addHitsAndRatings(ratedSearchHits); return evalQueryQuality; } @@ -173,7 +150,7 @@ public class PrecisionAtN implements RankedListQualityMetric { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.startObject(NAME); - builder.field(SIZE_FIELD.getPreferredName(), this.n); + builder.field(RELEVANT_RATING_FIELD.getPreferredName(), this.relevantRatingThreshhold); builder.endObject(); builder.endObject(); return builder; @@ -187,13 +164,13 @@ public class PrecisionAtN implements RankedListQualityMetric { if (obj == null || getClass() != obj.getClass()) { return false; } - PrecisionAtN other = (PrecisionAtN) obj; - return Objects.equals(n, other.n); + Precision other = (Precision) obj; + return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold); } @Override public final int hashCode() { - return Objects.hash(n); + return Objects.hash(relevantRatingThreshhold); } public static class Breakdown implements MetricDetails { @@ -247,7 +224,7 @@ public class PrecisionAtN implements RankedListQualityMetric { if (obj == null || getClass() != obj.getClass()) { return false; } - PrecisionAtN.Breakdown other = (PrecisionAtN.Breakdown) obj; + Precision.Breakdown other = (Precision.Breakdown) obj; return Objects.equals(relevantRetrieved, other.relevantRetrieved) && Objects.equals(retrieved, other.retrieved); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index c741f59f74c..c2261646867 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -50,12 +50,12 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin { @Override public List getNamedWriteables() { List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, PrecisionAtN.NAME, PrecisionAtN::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGainAt.NAME, DiscountedCumulativeGainAt::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, PrecisionAtN.NAME, - PrecisionAtN.Breakdown::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, Precision.NAME, + Precision.Breakdown::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, ReciprocalRank.NAME, ReciprocalRank.Breakdown::new)); return namedWriteables; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index e42d88a49da..ae25f0e2b70 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -64,8 +64,8 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { // TODO maybe switch to using a plugable registry later? switch (metricName) { - case PrecisionAtN.NAME: - rc = PrecisionAtN.fromXContent(parser, context); + case Precision.NAME: + rc = Precision.fromXContent(parser, context); break; case ReciprocalRank.NAME: rc = ReciprocalRank.fromXContent(parser, context); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java index 09e8c9a673a..276dbffda73 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -44,7 +44,7 @@ public class EvalQueryQualityTests extends ESTestCase { EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAsciiOfLength(10), randomDoubleBetween(0.0, 1.0, true)); if (randomBoolean()) { // TODO randomize this - evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(1, 5)); + evalQueryQuality.addMetricDetails(new Precision.Breakdown(1, 5)); } evalQueryQuality.addHitsAndRatings(ratedHits); return evalQueryQuality; @@ -78,7 +78,7 @@ public class EvalQueryQualityTests extends ESTestCase { break; case 2: if (metricDetails == null) { - metricDetails = new PrecisionAtN.Breakdown(1, 5); + metricDetails = new Precision.Breakdown(1, 5); } else { metricDetails = null; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java similarity index 69% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java index 800a38c0ec9..5854d4cab1a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java @@ -24,7 +24,7 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; -import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; +import org.elasticsearch.index.rankeval.Precision.Rating; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; @@ -36,15 +36,15 @@ import java.util.Collections; import java.util.List; import java.util.Vector; -public class PrecisionAtNTests extends ESTestCase { +public class PrecisionTests extends ESTestCase { public void testPrecisionAtFiveCalculation() { List rated = new ArrayList<>(); rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated, "test", "testtype"), rated); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated, "test", "testtype"), rated); assertEquals(1, evaluated.getQualityLevel(), 0.00001); - assertEquals(1, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(1, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(1, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(1, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveIgnoreOneResult() { @@ -54,10 +54,10 @@ public class PrecisionAtNTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); - EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated, "test", "testtype"), rated); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated, "test", "testtype"), rated); assertEquals((double) 4 / 5, evaluated.getQualityLevel(), 0.00001); - assertEquals(4, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(5, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(4, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } /** @@ -71,12 +71,12 @@ public class PrecisionAtNTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "2", 2)); rated.add(new RatedDocument("test", "testtype", "3", 3)); rated.add(new RatedDocument("test", "testtype", "4", 4)); - PrecisionAtN precisionAtN = new PrecisionAtN(5); + Precision precisionAtN = new Precision(); precisionAtN.setRelevantRatingThreshhold(2); EvalQueryQuality evaluated = precisionAtN.evaluate("id", toSearchHits(rated, "test", "testtype"), rated); assertEquals((double) 3 / 5, evaluated.getQualityLevel(), 0.00001); - assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(5, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveCorrectIndex() { @@ -87,10 +87,10 @@ public class PrecisionAtNTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "2", Rating.IRRELEVANT.ordinal())); // the following search hits contain only the last three documents - EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveCorrectType() { @@ -100,33 +100,35 @@ public class PrecisionAtNTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "2", Rating.IRRELEVANT.ordinal())); - EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testNoRatedDocs() throws Exception { - List rated = new ArrayList<>(); - EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated, "test", "testtype"), rated); + InternalSearchHit[] hits = new InternalSearchHit[5]; + for (int i = 0; i < 5; i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text("type"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); + } + EvalQueryQuality evaluated = (new Precision()).evaluate("id", hits, Collections.emptyList()); assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); - assertEquals(0, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(0, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testParseFromXContent() throws IOException { String xContent = " {\n" - + " \"size\": 10,\n" + " \"relevant_rating_threshold\" : 2" + "}"; XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent); - PrecisionAtN precicionAt = PrecisionAtN.fromXContent(parser, () -> ParseFieldMatcher.STRICT); - assertEquals(10, precicionAt.getN()); + Precision precicionAt = Precision.fromXContent(parser, () -> ParseFieldMatcher.STRICT); assertEquals(2, precicionAt.getRelevantRatingThreshold()); } public void testCombine() { - PrecisionAtN metric = new PrecisionAtN(); + Precision metric = new Precision(); Vector partialResults = new Vector<>(3); partialResults.add(new EvalQueryQuality("a", 0.1)); partialResults.add(new EvalQueryQuality("b", 0.2)); @@ -134,17 +136,20 @@ public class PrecisionAtNTests extends ESTestCase { assertEquals(0.3, metric.combine(partialResults), Double.MIN_VALUE); } - public static PrecisionAtN createTestItem() { - int position = randomIntBetween(0, 1000); - return new PrecisionAtN(position); + public static Precision createTestItem() { + Precision precision = new Precision(); + if (randomBoolean()) { + precision.setRelevantRatingThreshhold(randomIntBetween(0, 10)); + } + return precision; } public void testXContentRoundtrip() throws IOException { - PrecisionAtN testItem = createTestItem(); + Precision testItem = createTestItem(); XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); - PrecisionAtN parsedItem = PrecisionAtN.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); + Precision parsedItem = Precision.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index 93d88fc3c72..b250e9f9cfb 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -22,7 +22,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; +import org.elasticsearch.index.rankeval.Precision.Rating; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; @@ -84,7 +84,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { berlinRequest.setSummaryFields(Arrays.asList(new String[]{ "text", "title" })); specifications.add(berlinRequest); - RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtN(10)); + RankEvalSpec task = new RankEvalSpec(specifications, new Precision()); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); @@ -140,7 +140,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { brokenQuery.query(brokenRangeQuery); specifications.add(new RatedRequest("broken_query", brokenQuery, indices, types, createRelevant("1"))); - RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtN(10)); + RankEvalSpec task = new RankEvalSpec(specifications, new Precision()); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index a2e74886f64..c5685d18a15 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ParseFieldRegistry; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; @@ -57,7 +56,7 @@ public class RankEvalSpecTests extends ESTestCase { * setup for the whole base test class */ @BeforeClass - public static void init() throws IOException { + public static void init() { AggregatorParsers aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"), new ParseFieldRegistry<>("aggregation_pipes")); searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); @@ -92,22 +91,19 @@ public class RankEvalSpecTests extends ESTestCase { RankedListQualityMetric metric; if (randomBoolean()) { - metric = PrecisionAtNTests.createTestItem(); + metric = PrecisionTests.createTestItem(); } else { metric = DiscountedCumulativeGainAtTests.createTestItem(); } RankEvalSpec testItem = new RankEvalSpec(specs, metric); - XContentType contentType = ESTestCase.randomFrom(XContentType.values()); - XContent xContent = contentType.xContent(); - if (randomBoolean()) { final Map params = randomBoolean() ? null : Collections.singletonMap("key", "value"); ScriptType scriptType = randomFrom(ScriptType.values()); String script; if (scriptType == ScriptType.INLINE) { - try (XContentBuilder builder = XContentBuilder.builder(xContent)) { + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { builder.startObject(); builder.field("field", randomAsciiOfLengthBetween(1, 5)); builder.endObject(); @@ -122,15 +118,10 @@ public class RankEvalSpecTests extends ESTestCase { scriptType, randomFrom("_lang1", "_lang2", null), params, - scriptType == ScriptType.INLINE ? xContent.type() : null)); + scriptType == ScriptType.INLINE ? XContentType.JSON : null)); } - XContentBuilder builder = XContentFactory.contentBuilder(contentType); - if (ESTestCase.randomBoolean()) { - builder.prettyPrint(); - } - testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); - XContentBuilder shuffled = ESTestCase.shuffleXContent(builder); + XContentBuilder shuffled = ESTestCase.shuffleXContent(testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 694d478ee41..763e6d22365 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -23,7 +23,7 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; -import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; +import org.elasticsearch.index.rankeval.Precision.Rating; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index da6646cb0a5..129925ef8ed 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -56,7 +56,7 @@ "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] } ], - "metric" : { "precision_atn": { "size": 10}} + "metric" : { "precision": { }} } - match: { rank_eval.quality_level: 1} From c8d9d063ca21f9c755b0fbe90d594d46fcf0eca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 3 Nov 2016 17:33:39 +0100 Subject: [PATCH 068/297] Removing the 'size' parameter from the dcg metric --- ...nAt.java => DiscountedCumulativeGain.java} | 68 ++++--------------- .../index/rankeval/RankEvalPlugin.java | 4 +- .../rankeval/RankedListQualityMetric.java | 4 +- .../index/rankeval/ReciprocalRank.java | 2 +- ...ava => DiscountedCumulativeGainTests.java} | 44 ++++++------ .../index/rankeval/RankEvalSpecTests.java | 2 +- .../index/rankeval/ReciprocalRankTests.java | 3 +- .../rest-api-spec/test/rank_eval/20_dcg.yaml | 6 +- 8 files changed, 47 insertions(+), 86 deletions(-) rename modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/{DiscountedCumulativeGainAt.java => DiscountedCumulativeGain.java} (70%) rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{DiscountedCumulativeGainAtTests.java => DiscountedCumulativeGainTests.java} (86%) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java similarity index 70% rename from modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java rename to modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java index 4f5718a702e..a82e8ad65ff 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java @@ -38,56 +38,36 @@ import java.util.stream.Collectors; import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; -public class DiscountedCumulativeGainAt implements RankedListQualityMetric { +public class DiscountedCumulativeGain implements RankedListQualityMetric { - /** rank position up to which to check results. */ - private int position; /** If set to true, the dcg will be normalized (ndcg) */ private boolean normalize; /** If set to, this will be the rating for docs the user hasn't supplied an explicit rating for */ private Integer unknownDocRating; - public static final String NAME = "dcg_at_n"; + public static final String NAME = "dcg"; private static final double LOG2 = Math.log(2.0); - /** - * Initialises position with 10 - * */ - public DiscountedCumulativeGainAt() { - this.position = 10; + public DiscountedCumulativeGain() { } /** - * @param position number of top results to check against a given set of relevant results. Must be positive. - */ - public DiscountedCumulativeGainAt(int position) { - if (position <= 0) { - throw new IllegalArgumentException("number of results to check needs to be positive but was " + position); - } - this.position = position; - } - - /** - * @param position number of top results to check against a given set of relevant results. Must be positive. * @param normalize If set to true, dcg will be normalized (ndcg) * See https://en.wikipedia.org/wiki/Discounted_cumulative_gain * @param unknownDocRating the rating for docs the user hasn't supplied an explicit rating for * */ - public DiscountedCumulativeGainAt(int position, boolean normalize, Integer unknownDocRating) { - this(position); + public DiscountedCumulativeGain( boolean normalize, Integer unknownDocRating) { this.normalize = normalize; this.unknownDocRating = unknownDocRating; } - public DiscountedCumulativeGainAt(StreamInput in) throws IOException { - this(in.readInt()); + public DiscountedCumulativeGain(StreamInput in) throws IOException { normalize = in.readBoolean(); unknownDocRating = in.readOptionalVInt(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeInt(position); out.writeBoolean(normalize); out.writeOptionalVInt(unknownDocRating); } @@ -97,20 +77,6 @@ public class DiscountedCumulativeGainAt implements RankedListQualityMetric { return NAME; } - /** - * Return number of search results to check for quality metric. - */ - public int getPosition() { - return this.position; - } - - /** - * set number of search results to check for quality metric. - */ - public void setPosition(int position) { - this.position = position; - } - /** * If set to true, the dcg will be normalized (ndcg) */ @@ -143,8 +109,8 @@ public class DiscountedCumulativeGainAt implements RankedListQualityMetric { public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { List allRatings = ratedDocs.stream().mapToInt(RatedDocument::getRating).boxed().collect(Collectors.toList()); List ratedHits = joinHitsWithRatings(hits, ratedDocs); - List ratingsInSearchHits = new ArrayList<>(Math.min(ratedHits.size(), position)); - for (RatedSearchHit hit : ratedHits.subList(0, position)) { + List ratingsInSearchHits = new ArrayList<>(ratedHits.size()); + for (RatedSearchHit hit : ratedHits) { // unknownDocRating might be null, which means it will be unrated docs are ignored in the dcg calculation // we still need to add them as a placeholder so the rank of the subsequent ratings is correct ratingsInSearchHits.add(hit.getRating().orElse(unknownDocRating)); @@ -173,19 +139,17 @@ public class DiscountedCumulativeGainAt implements RankedListQualityMetric { return dcg; } - private static final ParseField SIZE_FIELD = new ParseField("size"); private static final ParseField NORMALIZE_FIELD = new ParseField("normalize"); private static final ParseField UNKNOWN_DOC_RATING_FIELD = new ParseField("unknown_doc_rating"); - private static final ObjectParser PARSER = - new ObjectParser<>("dcg_at", () -> new DiscountedCumulativeGainAt()); + private static final ObjectParser PARSER = + new ObjectParser<>("dcg_at", () -> new DiscountedCumulativeGain()); static { - PARSER.declareInt(DiscountedCumulativeGainAt::setPosition, SIZE_FIELD); - PARSER.declareBoolean(DiscountedCumulativeGainAt::setNormalize, NORMALIZE_FIELD); - PARSER.declareInt(DiscountedCumulativeGainAt::setUnknownDocRating, UNKNOWN_DOC_RATING_FIELD); + PARSER.declareBoolean(DiscountedCumulativeGain::setNormalize, NORMALIZE_FIELD); + PARSER.declareInt(DiscountedCumulativeGain::setUnknownDocRating, UNKNOWN_DOC_RATING_FIELD); } - public static DiscountedCumulativeGainAt fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { + public static DiscountedCumulativeGain fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } @@ -193,7 +157,6 @@ public class DiscountedCumulativeGainAt implements RankedListQualityMetric { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.startObject(NAME); - builder.field(SIZE_FIELD.getPreferredName(), this.position); builder.field(NORMALIZE_FIELD.getPreferredName(), this.normalize); if (unknownDocRating != null) { builder.field(UNKNOWN_DOC_RATING_FIELD.getPreferredName(), this.unknownDocRating); @@ -211,15 +174,14 @@ public class DiscountedCumulativeGainAt implements RankedListQualityMetric { if (obj == null || getClass() != obj.getClass()) { return false; } - DiscountedCumulativeGainAt other = (DiscountedCumulativeGainAt) obj; - return Objects.equals(position, other.position) && - Objects.equals(normalize, other.normalize) && + DiscountedCumulativeGain other = (DiscountedCumulativeGain) obj; + return Objects.equals(normalize, other.normalize) && Objects.equals(unknownDocRating, other.unknownDocRating); } @Override public final int hashCode() { - return Objects.hash(position, normalize, unknownDocRating); + return Objects.hash(normalize, unknownDocRating); } // TODO maybe also add debugging breakdown here diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index c2261646867..d4d6a3a22ea 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -52,8 +52,8 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin { List namedWriteables = new ArrayList<>(); namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGainAt.NAME, - DiscountedCumulativeGainAt::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, + DiscountedCumulativeGain::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, Precision.NAME, Precision.Breakdown::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, ReciprocalRank.NAME, diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index ae25f0e2b70..cd1f6cb0e85 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -70,8 +70,8 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { case ReciprocalRank.NAME: rc = ReciprocalRank.fromXContent(parser, context); break; - case DiscountedCumulativeGainAt.NAME: - rc = DiscountedCumulativeGainAt.fromXContent(parser, context); + case DiscountedCumulativeGain.NAME: + rc = DiscountedCumulativeGain.fromXContent(parser, context); break; default: throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index caf52c0e17c..160cc0aee77 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -45,7 +45,7 @@ import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsW public class ReciprocalRank implements RankedListQualityMetric { public static final String NAME = "reciprocal_rank"; - public static final int DEFAULT_MAX_ACCEPTABLE_RANK = 10; + public static final int DEFAULT_MAX_ACCEPTABLE_RANK = Integer.MAX_VALUE; private int maxAcceptableRank = DEFAULT_MAX_ACCEPTABLE_RANK; /** ratings equal or above this value will be considered relevant. */ diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java similarity index 86% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 9c5a5e3bb14..32a4d0dd82a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -30,14 +30,12 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.concurrent.ExecutionException; import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; -public class DiscountedCumulativeGainAtTests extends ESTestCase { +public class DiscountedCumulativeGainTests extends ESTestCase { /** * Assuming the docs are ranked in the following order: @@ -53,7 +51,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { * * dcg = 13.84826362927298 (sum of last column) */ - public void testDCGAt() throws IOException, InterruptedException, ExecutionException { + public void testDCGAt() { List rated = new ArrayList<>(); int[] relevanceRatings = new int[] { 3, 2, 3, 0, 1, 2 }; InternalSearchHit[] hits = new InternalSearchHit[6]; @@ -62,7 +60,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } - DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(6); + DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); assertEquals(13.84826362927298, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); /** @@ -97,7 +95,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { * * dcg = 12.779642067948913 (sum of last column) */ - public void testDCGAtSixMissingRatings() throws IOException, InterruptedException, ExecutionException { + public void testDCGAtSixMissingRatings() { List rated = new ArrayList<>(); Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1}; InternalSearchHit[] hits = new InternalSearchHit[6]; @@ -110,7 +108,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } - DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(6); + DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); EvalQueryQuality result = dcg.evaluate("id", hits, rated); assertEquals(12.779642067948913, result.getQualityLevel(), 0.00001); assertEquals(2, filterUnknownDocuments(result.getHitsAndRatings()).size()); @@ -149,21 +147,24 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { * * dcg = 12.392789260714371 (sum of last column until position 4) */ - public void testDCGAtFourMoreRatings() throws IOException, InterruptedException, ExecutionException { - List rated = new ArrayList<>(); + public void testDCGAtFourMoreRatings() { Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1, null}; - InternalSearchHit[] hits = new InternalSearchHit[6]; + List ratedDocs = new ArrayList<>(); for (int i = 0; i < 6; i++) { if (i < relevanceRatings.length) { if (relevanceRatings[i] != null) { - rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); + ratedDocs.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); } } + } + // only create four hits + InternalSearchHit[] hits = new InternalSearchHit[4]; + for (int i = 0; i < 4; i++) { hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } - DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(4); - EvalQueryQuality result = dcg.evaluate("id", Arrays.copyOfRange(hits, 0, 4), rated); + DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); + EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs); assertEquals(12.392789260714371 , result.getQualityLevel(), 0.00001); assertEquals(1, filterUnknownDocuments(result.getHitsAndRatings()).size()); @@ -183,33 +184,32 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { * idcg = 13.347184833073591 (sum of last column) */ dcg.setNormalize(true); - assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); + assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), 0.00001); } public void testParseFromXContent() throws IOException { String xContent = " {\n" - + " \"size\": 8,\n" + + " \"unknown_doc_rating\": 2,\n" + " \"normalize\": true\n" + "}"; XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent); - DiscountedCumulativeGainAt dcgAt = DiscountedCumulativeGainAt.fromXContent(parser, () -> ParseFieldMatcher.STRICT); - assertEquals(8, dcgAt.getPosition()); + DiscountedCumulativeGain dcgAt = DiscountedCumulativeGain.fromXContent(parser, () -> ParseFieldMatcher.STRICT); + assertEquals(2, dcgAt.getUnknownDocRating().intValue()); assertEquals(true, dcgAt.getNormalize()); } - public static DiscountedCumulativeGainAt createTestItem() { - int position = randomIntBetween(0, 1000); + public static DiscountedCumulativeGain createTestItem() { boolean normalize = randomBoolean(); Integer unknownDocRating = new Integer(randomIntBetween(0, 1000)); - return new DiscountedCumulativeGainAt(position, normalize, unknownDocRating); + return new DiscountedCumulativeGain(normalize, unknownDocRating); } public void testXContentRoundtrip() throws IOException { - DiscountedCumulativeGainAt testItem = createTestItem(); + DiscountedCumulativeGain testItem = createTestItem(); XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); - DiscountedCumulativeGainAt parsedItem = DiscountedCumulativeGainAt.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); + DiscountedCumulativeGain parsedItem = DiscountedCumulativeGain.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index c5685d18a15..b7a6746d965 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -93,7 +93,7 @@ public class RankEvalSpecTests extends ESTestCase { if (randomBoolean()) { metric = PrecisionTests.createTestItem(); } else { - metric = DiscountedCumulativeGainAtTests.createTestItem(); + metric = DiscountedCumulativeGainTests.createTestItem(); } RankEvalSpec testItem = new RankEvalSpec(specs, metric); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 763e6d22365..9220d501c84 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Vector; -import java.util.concurrent.ExecutionException; public class ReciprocalRankTests extends ESTestCase { @@ -98,7 +97,7 @@ public class ReciprocalRankTests extends ESTestCase { * e.g. we set it to 2 here and expect dics 0-2 to be not relevant, so first relevant doc has * third ranking position, so RR should be 1/3 */ - public void testPrecisionAtFiveRelevanceThreshold() throws IOException, InterruptedException, ExecutionException { + public void testPrecisionAtFiveRelevanceThreshold() { List rated = new ArrayList<>(); rated.add(new RatedDocument("test", "testtype", "0", 0)); rated.add(new RatedDocument("test", "testtype", "1", 1)); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml index 8310389c02a..2f784790ad5 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml @@ -60,7 +60,7 @@ {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] } ], - "metric" : { "dcg_at_n": { "size": 6}} + "metric" : { "dcg": {}} } - match: {rank_eval.quality_level: 13.84826362927298} @@ -85,7 +85,7 @@ {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] }, ], - "metric" : { "dcg_at_n": { "size": 6}} + "metric" : { "dcg": { }} } - match: {rank_eval.quality_level: 10.29967439154499} @@ -121,7 +121,7 @@ {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] }, ], - "metric" : { "dcg_at_n": { "size": 6}} + "metric" : { "dcg": { }} } - match: {rank_eval.quality_level: 12.073969010408984} From adb24333caf72e7bc3e99eecc34f2ca5f9cd9070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 8 Nov 2016 12:17:36 +0100 Subject: [PATCH 069/297] Remove maxAcceptableRank parameter from ReciprocalRank --- .../index/rankeval/ReciprocalRank.java | 42 +++---------------- .../index/rankeval/ReciprocalRankTests.java | 35 ++++++---------- 2 files changed, 19 insertions(+), 58 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 160cc0aee77..58cc8a3f100 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -45,8 +45,6 @@ import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsW public class ReciprocalRank implements RankedListQualityMetric { public static final String NAME = "reciprocal_rank"; - public static final int DEFAULT_MAX_ACCEPTABLE_RANK = Integer.MAX_VALUE; - private int maxAcceptableRank = DEFAULT_MAX_ACCEPTABLE_RANK; /** ratings equal or above this value will be considered relevant. */ private int relevantRatingThreshhold = 1; @@ -58,19 +56,8 @@ public class ReciprocalRank implements RankedListQualityMetric { // use defaults } - /** - * @param maxAcceptableRank - * maximal acceptable rank. Must be positive. - */ - public ReciprocalRank(int maxAcceptableRank) { - if (maxAcceptableRank <= 0) { - throw new IllegalArgumentException("maximal acceptable rank needs to be positive but was [" + maxAcceptableRank + "]"); - } - this.maxAcceptableRank = maxAcceptableRank; - } - public ReciprocalRank(StreamInput in) throws IOException { - this.maxAcceptableRank = in.readInt(); + this.relevantRatingThreshhold = in.readInt(); } @Override @@ -78,21 +65,6 @@ public class ReciprocalRank implements RankedListQualityMetric { return NAME; } - /** - * @param maxAcceptableRank - * maximal acceptable rank. Must be positive. - */ - public void setMaxAcceptableRank(int maxAcceptableRank) { - if (maxAcceptableRank <= 0) { - throw new IllegalArgumentException("maximal acceptable rank needs to be positive but was [" + maxAcceptableRank + "]"); - } - this.maxAcceptableRank = maxAcceptableRank; - } - - public int getMaxAcceptableRank() { - return this.maxAcceptableRank; - } - /** * Sets the rating threshold above which ratings are considered to be "relevant" for this metric. * */ @@ -117,7 +89,7 @@ public class ReciprocalRank implements RankedListQualityMetric { List ratedHits = joinHitsWithRatings(hits, ratedDocs); int firstRelevant = -1; int rank = 1; - for (RatedSearchHit hit : ratedHits.subList(0, Math.min(maxAcceptableRank, ratedHits.size()))) { + for (RatedSearchHit hit : ratedHits) { Optional rating = hit.getRating(); if (rating.isPresent()) { if (rating.get() >= this.relevantRatingThreshhold) { @@ -137,16 +109,14 @@ public class ReciprocalRank implements RankedListQualityMetric { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(maxAcceptableRank); + out.writeVInt(relevantRatingThreshhold); } - private static final ParseField MAX_RANK_FIELD = new ParseField("max_acceptable_rank"); private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); private static final ObjectParser PARSER = new ObjectParser<>( "reciprocal_rank", () -> new ReciprocalRank()); static { - PARSER.declareInt(ReciprocalRank::setMaxAcceptableRank, MAX_RANK_FIELD); PARSER.declareInt(ReciprocalRank::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); } @@ -158,7 +128,7 @@ public class ReciprocalRank implements RankedListQualityMetric { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.startObject(NAME); - builder.field(MAX_RANK_FIELD.getPreferredName(), this.maxAcceptableRank); + builder.field(RELEVANT_RATING_FIELD.getPreferredName(), this.relevantRatingThreshhold); builder.endObject(); builder.endObject(); return builder; @@ -173,12 +143,12 @@ public class ReciprocalRank implements RankedListQualityMetric { return false; } ReciprocalRank other = (ReciprocalRank) obj; - return Objects.equals(maxAcceptableRank, other.maxAcceptableRank); + return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold); } @Override public final int hashCode() { - return Objects.hash(maxAcceptableRank); + return Objects.hash(relevantRatingThreshhold); } public static class Breakdown implements MetricDetails { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 9220d501c84..3f00562d1e1 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Vector; @@ -39,16 +40,13 @@ public class ReciprocalRankTests extends ESTestCase { public void testMaxAcceptableRank() { ReciprocalRank reciprocalRank = new ReciprocalRank(); - assertEquals(ReciprocalRank.DEFAULT_MAX_ACCEPTABLE_RANK, reciprocalRank.getMaxAcceptableRank()); - int maxRank = randomIntBetween(1, 100); - reciprocalRank.setMaxAcceptableRank(maxRank); - assertEquals(maxRank, reciprocalRank.getMaxAcceptableRank()); + int searchHits = randomIntBetween(1, 50); - SearchHit[] hits = createSearchHits(0, 9, "test", "type"); + SearchHit[] hits = createSearchHits(0, searchHits, "test", "type"); List ratedDocs = new ArrayList<>(); - int relevantAt = 5; - for (int i = 0; i < 10; i++) { + int relevantAt = randomIntBetween(0, searchHits); + for (int i = 0; i <= searchHits; i++) { if (i == relevantAt) { ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), Rating.RELEVANT.ordinal())); } else { @@ -58,19 +56,13 @@ public class ReciprocalRankTests extends ESTestCase { int rankAtFirstRelevant = relevantAt + 1; EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); - if (rankAtFirstRelevant <= maxRank) { - assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE); - assertEquals(rankAtFirstRelevant, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); + assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE); + assertEquals(rankAtFirstRelevant, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); - // check that if we lower maxRank by one, we don't find any result and get 0.0 quality level - reciprocalRank = new ReciprocalRank(rankAtFirstRelevant - 1); - evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); - assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); - - } else { - assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); - assertEquals(-1, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); - } + // check that if we have fewer search hits than relevant doc position, we don't find any result and get 0.0 quality level + reciprocalRank = new ReciprocalRank(); + evaluation = reciprocalRank.evaluate("id", Arrays.copyOfRange(hits, 0, relevantAt), ratedDocs); + assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); } public void testEvaluationOneRelevantInResults() { @@ -131,9 +123,8 @@ public class ReciprocalRankTests extends ESTestCase { } public void testXContentRoundtrip() throws IOException { - int position = randomIntBetween(0, 1000); - - ReciprocalRank testItem = new ReciprocalRank(position); + ReciprocalRank testItem = new ReciprocalRank(); + testItem.setRelevantRatingThreshhold(randomIntBetween(0, 20)); XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); From 4718f000dfd61b12c739d4586d87b395e7ec7767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 8 Nov 2016 15:51:37 +0100 Subject: [PATCH 070/297] Remove test that uses max_acceptable_rank parameter --- .../test/rank_eval/10_basic.yaml | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 129925ef8ed..ea705bdd33d 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -152,37 +152,3 @@ - match: {rank_eval.details.berlin_query.metric_details: {"first_relevant": 2}} - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc1"}]} - - do: - rank_eval: - body: { - "requests" : [ - { - "id": "amsterdam_query", - "request": { "query": { "match" : {"text" : "amsterdam" }}}, - # doc4 should be returned in third position, so reciprocal rank is 1/3 - "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc4", "rating": 1}] - }, - { - "id" : "berlin_query", - "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, - # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 - "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc4", "rating": 1}] - } - ], - "metric" : { - "reciprocal_rank": { - # the following will make the first query have a quality value of 0.0 - "max_acceptable_rank" : 2 - } - } - } - -# average is (0 + 1/2)/2 = 1/4 - - match: {rank_eval.quality_level: 0.25} - - match: {rank_eval.details.amsterdam_query.quality_level: 0} - - match: {rank_eval.details.amsterdam_query.metric_details: {"first_relevant": -1}} - - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc2"}, - {"_index": "foo", "_type": "bar", "_id": "doc3"} ]} - - match: {rank_eval.details.berlin_query.quality_level: 0.5} - - match: {rank_eval.details.berlin_query.metric_details: {"first_relevant": 2}} - - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc1"}]} From 4e5c8687090b524873d787ec317a0050f4bfc696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 8 Nov 2016 15:36:59 +0100 Subject: [PATCH 071/297] RankEval: Add optional parameter to ignore unlabeled documents in Precision metric Our current default behaviour to ignore unrated documents when calculating the precision seems a bit counter intuitive. Instead we should treat those documents as "irrelevant" by default and provide an optional parameter to ignore those documents if that is the behaviour the user wants. --- .../index/rankeval/Precision.java | 63 ++++++++++--------- .../index/rankeval/PrecisionTests.java | 38 ++++++++++- .../index/rankeval/RankEvalRequestTests.java | 9 +-- .../index/rankeval/ReciprocalRankTests.java | 2 +- .../test/rank_eval/10_basic.yaml | 2 +- 5 files changed, 78 insertions(+), 36 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java index 4871a1e5359..f85e9460120 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java @@ -45,29 +45,40 @@ import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsW */ public class Precision implements RankedListQualityMetric { - /** ratings equal or above this value will be considered relevant. */ - private int relevantRatingThreshhold = 1; - public static final String NAME = "precision"; private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); + private static final ParseField IGNORE_UNLABELED_FIELD = new ParseField("ignore_unlabeled"); private static final ObjectParser PARSER = new ObjectParser<>(NAME, Precision::new); + /** + * This setting controls how unlabeled documents in the search hits are + * treated. Set to 'true', unlabeled documents are ignored and neither count + * as true or false positives. Set to 'false', they are treated as false positives. + */ + private boolean ignoreUnlabeled = false; + + /** ratings equal or above this value will be considered relevant. */ + private int relevantRatingThreshhold = 1; + public Precision() { // needed for supplier in parser } static { PARSER.declareInt(Precision::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); + PARSER.declareBoolean(Precision::setIgnoreUnlabeled, IGNORE_UNLABELED_FIELD); } public Precision(StreamInput in) throws IOException { relevantRatingThreshhold = in.readOptionalVInt(); + ignoreUnlabeled = in.readOptionalBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalVInt(relevantRatingThreshhold); + out.writeOptionalBoolean(ignoreUnlabeled); } @Override @@ -90,6 +101,20 @@ public class Precision implements RankedListQualityMetric { return relevantRatingThreshhold ; } + /** + * Sets the 'ìgnore_unlabeled' parameter + * */ + public void setIgnoreUnlabeled(boolean ignoreUnlabeled) { + this.ignoreUnlabeled = ignoreUnlabeled; + } + + /** + * Gets the 'ìgnore_unlabeled' parameter + * */ + public boolean getIgnoreUnlabeled() { + return ignoreUnlabeled; + } + public static Precision fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } @@ -110,6 +135,8 @@ public class Precision implements RankedListQualityMetric { } else { falsePositives++; } + } else if (ignoreUnlabeled == false) { + falsePositives++; } } double precision = 0.0; @@ -122,35 +149,12 @@ public class Precision implements RankedListQualityMetric { return evalQueryQuality; } - // TODO add abstraction that also works for other metrics - public enum Rating { - IRRELEVANT, RELEVANT; - } - - /** - * Needed to get the enum accross serialisation boundaries. - * */ - public static class RatingMapping { - public static Integer mapFrom(Rating rating) { - if (Rating.RELEVANT.equals(rating)) { - return 1; - } - return 0; - } - - public static Rating mapTo(Integer rating) { - if (rating == 1) { - return Rating.RELEVANT; - } - return Rating.IRRELEVANT; - } - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.startObject(NAME); builder.field(RELEVANT_RATING_FIELD.getPreferredName(), this.relevantRatingThreshhold); + builder.field(IGNORE_UNLABELED_FIELD.getPreferredName(), this.ignoreUnlabeled); builder.endObject(); builder.endObject(); return builder; @@ -165,12 +169,13 @@ public class Precision implements RankedListQualityMetric { return false; } Precision other = (Precision) obj; - return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold); + return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold) && + Objects.equals(ignoreUnlabeled, other.ignoreUnlabeled); } @Override public final int hashCode() { - return Objects.hash(relevantRatingThreshhold); + return Objects.hash(relevantRatingThreshhold, ignoreUnlabeled); } public static class Breakdown implements MetricDetails { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java index 5854d4cab1a..75c7818f40e 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; -import org.elasticsearch.index.rankeval.Precision.Rating; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; @@ -32,6 +31,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Vector; @@ -106,6 +106,29 @@ public class PrecisionTests extends ESTestCase { assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } + public void testIgnoreUnlabeled() { + List rated = new ArrayList<>(); + rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); + // add an unlabeled search hit + SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test", "testtype"), 3); + searchHits[2] = new InternalSearchHit(2, "2", new Text("testtype"), Collections.emptyMap()); + ((InternalSearchHit)searchHits[2]).shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); + + EvalQueryQuality evaluated = (new Precision()).evaluate("id", searchHits, rated); + assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); + assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + + // also try with setting `ignore_unlabeled` + Precision prec = new Precision(); + prec.setIgnoreUnlabeled(true); + evaluated = prec.evaluate("id", searchHits, rated); + assertEquals((double) 2 / 2, evaluated.getQualityLevel(), 0.00001); + assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + } + public void testNoRatedDocs() throws Exception { InternalSearchHit[] hits = new InternalSearchHit[5]; for (int i = 0; i < 5; i++) { @@ -115,6 +138,14 @@ public class PrecisionTests extends ESTestCase { EvalQueryQuality evaluated = (new Precision()).evaluate("id", hits, Collections.emptyList()); assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + + // also try with setting `ignore_unlabeled` + Precision prec = new Precision(); + prec.setIgnoreUnlabeled(true); + evaluated = prec.evaluate("id", hits, Collections.emptyList()); + assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); + assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } @@ -141,6 +172,7 @@ public class PrecisionTests extends ESTestCase { if (randomBoolean()) { precision.setRelevantRatingThreshhold(randomIntBetween(0, 10)); } + precision.setIgnoreUnlabeled(randomBoolean()); return precision; } @@ -163,4 +195,8 @@ public class PrecisionTests extends ESTestCase { } return hits; } + + public enum Rating { + IRRELEVANT, RELEVANT; + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index b250e9f9cfb..6ce18182fce 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -22,13 +22,12 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.index.rankeval.Precision.Rating; +import org.elasticsearch.index.rankeval.PrecisionTests.Rating; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.junit.Before; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -70,7 +69,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { refresh(); } - public void testPrecisionAtRequest() throws IOException { + public void testPrecisionAtRequest() { List indices = Arrays.asList(new String[] { "test" }); List types = Arrays.asList(new String[] { "testtype" }); @@ -84,7 +83,9 @@ public class RankEvalRequestTests extends ESIntegTestCase { berlinRequest.setSummaryFields(Arrays.asList(new String[]{ "text", "title" })); specifications.add(berlinRequest); - RankEvalSpec task = new RankEvalSpec(specifications, new Precision()); + Precision metric = new Precision(); + metric.setIgnoreUnlabeled(true); + RankEvalSpec task = new RankEvalSpec(specifications, metric); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 3f00562d1e1..4ce34ebc0d1 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -23,7 +23,7 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; -import org.elasticsearch.index.rankeval.Precision.Rating; +import org.elasticsearch.index.rankeval.PrecisionTests.Rating; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index ea705bdd33d..a0170f9722a 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -56,7 +56,7 @@ "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] } ], - "metric" : { "precision": { }} + "metric" : { "precision": { "ignore_unlabeled" : true }} } - match: { rank_eval.quality_level: 1} From 57ea1abb55487278a014171fde0c7fea5f2eed14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 11 Nov 2016 14:49:17 +0100 Subject: [PATCH 072/297] Fixing compile error in SmokeTestRankEvalWithMustacheYAMLTestSuiteIT --- .../elasticsearch/index/rankeval/DiscountedCumulativeGain.java | 2 +- .../smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java index a82e8ad65ff..65af4ec193a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java @@ -99,7 +99,7 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { } /** - * check whether this metric computes only dcg or "normalized" ndcg + * get the rating used for unrated documents */ public Integer getUnknownDocRating() { return this.unknownDocRating; diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java index 6fc2829c4a4..92c60b6f090 100644 --- a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java @@ -36,7 +36,7 @@ public class SmokeTestRankEvalWithMustacheYAMLTestSuiteIT extends ESClientYamlSu @ParametersFactory public static Iterable parameters() throws IOException, ClientYamlTestParseException { - return ESClientYamlSuiteTestCase.createParameters(0, 1); + return ESClientYamlSuiteTestCase.createParameters(); } } From 3d4804f9ed50a1512fe82436d8f0018790efeb5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 15 Nov 2016 15:55:09 +0100 Subject: [PATCH 073/297] Fix test after merging in master branch --- .../org/elasticsearch/index/rankeval/RankEvalSpecTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 62155d8bcfd..6694360a2e7 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -98,7 +98,7 @@ public class RankEvalSpecTests extends ESTestCase { RankEvalSpec testItem = new RankEvalSpec(specs, metric); if (randomBoolean()) { - final Map params = randomBoolean() ? null : Collections.singletonMap("key", "value"); + final Map params = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("key", "value"); ScriptType scriptType = randomFrom(ScriptType.values()); String script; if (scriptType == ScriptType.INLINE) { From 89155fd68cff9678647e1b9bbb410eabc465ef0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 15 Nov 2016 18:58:40 +0100 Subject: [PATCH 074/297] Fixing rank_eval qa test --- .../resources/rest-api-spec/test/rank-eval/30_template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml index bf94ea6ae8f..87bb6a843fc 100644 --- a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml @@ -59,7 +59,7 @@ "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] } ], - "metric" : { "precision_atn": { "size": 10}} + "metric" : { "precision": { }} } - match: {rank_eval.quality_level: 1} From 3ebb2265544fb32dc0c5a3087e00ea2e27d2c41c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 15 Nov 2016 19:43:01 +0100 Subject: [PATCH 075/297] Fixing wrong quality score in qa test --- .../resources/rest-api-spec/test/rank-eval/30_template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml index 87bb6a843fc..8ab994878a1 100644 --- a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml @@ -62,6 +62,6 @@ "metric" : { "precision": { }} } - - match: {rank_eval.quality_level: 1} + - match: {rank_eval.quality_level: 0.5833333333333333} - match: {rank_eval.details.berlin_query.unknown_docs.0._id: "doc4"} - match: {rank_eval.details.amsterdam_query.unknown_docs.0._id: "doc4"} From 6cc7fb5600d466cb41a6861cd5f224c5e843e992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 15 Nov 2016 20:53:51 +0100 Subject: [PATCH 076/297] Fix potential NPE in RankEvalSpec roundtrip test --- .../org/elasticsearch/index/rankeval/RankEvalSpecTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 6694360a2e7..6a344994a5a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -112,7 +112,7 @@ public class RankEvalSpecTests extends ESTestCase { script = randomAsciiOfLengthBetween(1, 5); } - testItem.setTemplate(new Script(scriptType, randomFrom("_lang1", "_lang2", null), script, params)); + testItem.setTemplate(new Script(scriptType, randomFrom("_lang1", "_lang2"), script, params)); } XContentBuilder shuffled = ESTestCase.shuffleXContent(testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); From 19bb0a928d9495b54704d5c9e7bee57159aa9189 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Thu, 17 Nov 2016 10:27:57 +0100 Subject: [PATCH 077/297] Reference documentation for rank evaluation API (#21427) * Reference documentation for rank evaluation API This adds a first page of reference documentation to the current state of the rank evaluation API. Closes to #21402 * Add default values for precision metric. Add information on default relevant_rating_threshold and ignore_unlabeled settings. Relates to #21304 * Move under search request docs, fix formatting Also removes some detail where it seemed unneeded for reference docs --- docs/reference/search.asciidoc | 2 + docs/reference/search/rank-eval.asciidoc | 197 +++++++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 docs/reference/search/rank-eval.asciidoc diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc index 7a26ee2cf60..64c5963c87c 100644 --- a/docs/reference/search.asciidoc +++ b/docs/reference/search.asciidoc @@ -131,4 +131,6 @@ include::search/profile.asciidoc[] include::search/percolate.asciidoc[] +include::search/rank-eval.asciidoc[] + include::search/field-stats.asciidoc[] diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc new file mode 100644 index 00000000000..f22916815e1 --- /dev/null +++ b/docs/reference/search/rank-eval.asciidoc @@ -0,0 +1,197 @@ +[[rank-eval]] += Ranking Evaluation + +[partintro] +-- + +Imagine having built and deployed a search application: Users are happily +entering queries into your search frontend. Your application takes these +queries and creates a dedicated Elasticsearch query from that, and returns its +results back to the user. Imagine further that you are tasked with tweaking the +Elasticsearch query that is being created to return specific results for a +certain set of queries without breaking others. How should that be done? + +One possible solution is to gather a sample of user queries representative of +how the search application is used, retrieve the search results that are being +returned. As a next step these search results would be manually annotated for +their relevancy to the original user query. Based on this set of rated requests +we can compute a couple of metrics telling us more about how many relevant +search results are being returned. + +This is a nice approximation for how well our translation from user query to +Elasticsearch query works for providing the user with relevant search results. +Elasticsearch provides a ranking evaluation API that lets you compute scores for +your current ranking function based on annotated search results. +-- + +== Plain ranking evaluation + +In its most simple form, for each query a set of ratings can be supplied: + +[source,js] +----------------------------- +GET /index/type/_rank_eval +{ + "requests": [ + { + "id": "JFK query", <1> + "request": { + "query": { + "match": { + "opening_text": { + "query": "JFK"}}}}, <2> + "ratings": [ <3> + { + "rating": 1.5, <4> + "_type": "page", <5> + "_id": "13736278", <6> + "_index": "enwiki_rank" <7> + }, + { + "rating": 1, + "_type": "page", + "_id": "30900421", + "_index": "enwiki_rank" + }], + "summary_fields": ["title"], <8> + }, + "metric": { <9> + "reciprocal_rank": {} + } +} +------------------------------ +// CONSOLE + +<1> A human readable id for the rated query (that will be re-used in the response to provide further details). +<2> The actual Elasticsearch query to execute. +<3> A set of ratings for how well a certain document fits as response for the query. +<4> A rating expressing how well the document fits the query, higher is better, are treated as int values. +<5> The type where the rated document lives. +<6> The id of the rated document. +<7> The index where the rated document lives. +<8> For a verbose response, specify which properties of a search hit should be returned in addition to index/type/id. +<9> A metric to use for evaluation. See below for a list. + + +== Template based ranking evaluation + +[source,js] +-------------------------------- +GET /index/type/_rank_eval/template +{ + "template": { + "inline": { + "query": { + "match": { + "{{wiki_field}}": { + "query": "{{query_string}}"}}}}}, <1> + "requests": [ + { + "id": "JFK query" + "ratings": [ + { + "rating": 1.5, + "_type": "page", + "_id": "13736278", + "_index": "enwiki_rank" + }, + { + "rating": 1, + "_type": "page", + "_id": "30900421", + "_index": "enwiki_rank" + } ], + "params": { + "query_string": "JFK", <2> + "wiki_field": "opening_text" <2> + }, + }], + "metric": { + "precision": { + "relevant_rating_threshold": 2 + } + } +} +-------------------------------- +// CONSOLE + +<1> The template to use for every rated search request. +<2> The parameters to use to fill the template above. + + +== Valid evaluation metrics + +=== Precision + +Citing from https://en.wikipedia.org/wiki/Information_retrieval#Precision[Precision +page at Wikipedia]: +"Precision is the fraction of the documents retrieved that are relevant to the +user's information need." + +Works well as an easy to explain evaluation metric. Caveat: All result positions +are treated equally. So a ranking of ten results that contains one relevant +result in position 10 is equally good as a ranking of ten results that contains +one relevant result in position 1. + +[source,js] +-------------------------------- +{ + "metric": { + "precision": { + "relevant_rating_threshold": 1, <1> + "ignore_unlabeled": "false" <2> + } + } +} +-------------------------------- + +<1> For graded relevance ratings only ratings above this threshold are +considered as relevant results for the given query. By default this is set to 1. + +<2> All documents retrieved by the rated request that have no ratings +assigned are treated unrelevant by default. Set to true in order to drop them +from the precision computation entirely. + + +=== Reciprocal rank + +For any given query this is the reciprocal of the rank of the +first relevant document retrieved. For example finding the first relevant result +in position 3 means Reciprocal Rank is going to be 1/3. + +[source,js] +-------------------------------- +{ + "metric": { + "reciprocal_rank": {} + } +} +-------------------------------- + + +=== Normalized discounted cumulative gain + +In contrast to the two metrics above this takes both, the grade of the result +found as well as the position of the document returned into account. + +For more details also check the explanation on +https://en.wikipedia.org/wiki/Discounted_cumulative_gain[Wikipedia]. + + +[source,js] +-------------------------------- +{ + "metric": { + "dcg": { + "normalize": "false" <1> + } + } +} +-------------------------------- + +<1> Set to true to compute nDCG instead of DCG, default is false. + +Setting normalize to true makes DCG values better comparable across different +result set sizes. See also +https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG[Wikipedia +nDCG] for more details. From e5a08937e3e9dbdb17c0b33218bce0f5e16e7612 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Thu, 17 Nov 2016 15:01:04 +0100 Subject: [PATCH 078/297] Fix failing doc test, add missing CONSOLE --- docs/reference/search/rank-eval.asciidoc | 37 +++++++++++++++++++----- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index f22916815e1..549929b0b9a 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -53,8 +53,8 @@ GET /index/type/_rank_eval "_id": "30900421", "_index": "enwiki_rank" }], - "summary_fields": ["title"], <8> - }, + "summary_fields": ["title"] <8> + }], "metric": { <9> "reciprocal_rank": {} } @@ -87,7 +87,7 @@ GET /index/type/_rank_eval/template "query": "{{query_string}}"}}}}}, <1> "requests": [ { - "id": "JFK query" + "id": "JFK query", "ratings": [ { "rating": 1.5, @@ -100,11 +100,11 @@ GET /index/type/_rank_eval/template "_type": "page", "_id": "30900421", "_index": "enwiki_rank" - } ], + }], "params": { "query_string": "JFK", <2> "wiki_field": "opening_text" <2> - }, + } }], "metric": { "precision": { @@ -135,7 +135,14 @@ one relevant result in position 1. [source,js] -------------------------------- +GET /index/type/_rank_eval { + "requests": [ + { + "id": "JFK query", + "request": { "query": { "match_all": {}}}, + "ratings": [] + }], "metric": { "precision": { "relevant_rating_threshold": 1, <1> @@ -144,6 +151,7 @@ one relevant result in position 1. } } -------------------------------- +// CONSOLE <1> For graded relevance ratings only ratings above this threshold are considered as relevant results for the given query. By default this is set to 1. @@ -161,13 +169,20 @@ in position 3 means Reciprocal Rank is going to be 1/3. [source,js] -------------------------------- +GET /index/type/_rank_eval { + "requests": [ + { + "id": "JFK query", + "request": { "query": { "match_all": {}}}, + "ratings": [] + }], "metric": { "reciprocal_rank": {} } } -------------------------------- - +// CONSOLE === Normalized discounted cumulative gain @@ -180,14 +195,22 @@ https://en.wikipedia.org/wiki/Discounted_cumulative_gain[Wikipedia]. [source,js] -------------------------------- +GET /index/type/_rank_eval { + "requests": [ + { + "id": "JFK query", + "request": { "query": { "match_all": {}}}, + "ratings": [] + }], "metric": { "dcg": { - "normalize": "false" <1> + "normalize": false <1> } } } -------------------------------- +// CONSOLE <1> Set to true to compute nDCG instead of DCG, default is false. From 9c58578dc6afd030f0e042f96ea75c7434bbc69d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 17 Nov 2016 15:18:08 +0100 Subject: [PATCH 079/297] Renaming RankEvalRequestTests to RankEvalRequestIT --- .../{RankEvalRequestTests.java => RankEvalRequestIT.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{RankEvalRequestTests.java => RankEvalRequestIT.java} (99%) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java similarity index 99% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 6ce18182fce..ab24fdbb937 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -38,7 +38,7 @@ import java.util.Set; import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; -public class RankEvalRequestTests extends ESIntegTestCase { +public class RankEvalRequestIT extends ESIntegTestCase { @Override protected Collection> transportClientPlugins() { return Arrays.asList(RankEvalPlugin.class); From 1f5f2e312ab8101efca35c0ca1cf1377600aac44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 17 Nov 2016 17:03:46 +0100 Subject: [PATCH 080/297] Use the twitter index in documentation snippets --- docs/reference/search/rank-eval.asciidoc | 39 +++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index 549929b0b9a..0220e48a5cd 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -30,7 +30,7 @@ In its most simple form, for each query a set of ratings can be supplied: [source,js] ----------------------------- -GET /index/type/_rank_eval +GET /twitter/tweet/_rank_eval { "requests": [ { @@ -38,20 +38,20 @@ GET /index/type/_rank_eval "request": { "query": { "match": { - "opening_text": { + "title": { "query": "JFK"}}}}, <2> "ratings": [ <3> { "rating": 1.5, <4> - "_type": "page", <5> + "_type": "tweet", <5> "_id": "13736278", <6> - "_index": "enwiki_rank" <7> + "_index": "twitter" <7> }, { "rating": 1, - "_type": "page", + "_type": "tweet", "_id": "30900421", - "_index": "enwiki_rank" + "_index": "twitter" }], "summary_fields": ["title"] <8> }], @@ -61,6 +61,7 @@ GET /index/type/_rank_eval } ------------------------------ // CONSOLE +// TEST[setup:twitter] <1> A human readable id for the rated query (that will be re-used in the response to provide further details). <2> The actual Elasticsearch query to execute. @@ -77,13 +78,13 @@ GET /index/type/_rank_eval [source,js] -------------------------------- -GET /index/type/_rank_eval/template +GET /twitter/tweet/_rank_eval/template { "template": { "inline": { "query": { "match": { - "{{wiki_field}}": { + "{{field}}": { "query": "{{query_string}}"}}}}}, <1> "requests": [ { @@ -91,19 +92,19 @@ GET /index/type/_rank_eval/template "ratings": [ { "rating": 1.5, - "_type": "page", + "_type": "tweet", "_id": "13736278", - "_index": "enwiki_rank" + "_index": "twitter" }, { "rating": 1, - "_type": "page", + "_type": "tweet", "_id": "30900421", - "_index": "enwiki_rank" + "_index": "twitter" }], "params": { "query_string": "JFK", <2> - "wiki_field": "opening_text" <2> + "field": "opening_text" <2> } }], "metric": { @@ -114,6 +115,7 @@ GET /index/type/_rank_eval/template } -------------------------------- // CONSOLE +// TEST[setup:twitter] <1> The template to use for every rated search request. <2> The parameters to use to fill the template above. @@ -135,7 +137,7 @@ one relevant result in position 1. [source,js] -------------------------------- -GET /index/type/_rank_eval +GET /twitter/tweet/_rank_eval { "requests": [ { @@ -146,12 +148,13 @@ GET /index/type/_rank_eval "metric": { "precision": { "relevant_rating_threshold": 1, <1> - "ignore_unlabeled": "false" <2> + "ignore_unlabeled": false <2> } } } -------------------------------- // CONSOLE +// TEST[setup:twitter] <1> For graded relevance ratings only ratings above this threshold are considered as relevant results for the given query. By default this is set to 1. @@ -169,7 +172,7 @@ in position 3 means Reciprocal Rank is going to be 1/3. [source,js] -------------------------------- -GET /index/type/_rank_eval +GET /twitter/tweet/_rank_eval { "requests": [ { @@ -183,6 +186,7 @@ GET /index/type/_rank_eval } -------------------------------- // CONSOLE +// TEST[setup:twitter] === Normalized discounted cumulative gain @@ -195,7 +199,7 @@ https://en.wikipedia.org/wiki/Discounted_cumulative_gain[Wikipedia]. [source,js] -------------------------------- -GET /index/type/_rank_eval +GET /twitter/tweet/_rank_eval { "requests": [ { @@ -211,6 +215,7 @@ GET /index/type/_rank_eval } -------------------------------- // CONSOLE +// TEST[setup:twitter] <1> Set to true to compute nDCG instead of DCG, default is false. From 8ba771e9132b8fee49fcf88e3ffe69511e27acf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 16 Nov 2016 13:04:57 +0100 Subject: [PATCH 081/297] Rank Eval: Handle exceptions on search requests and add them to response Currently we fail the whole ranking evaluation request when we receive an exception for any of the search requests. We should collect those errors and report them back to the user in the rest response. This change adds collecting the errors and propagating them back via the RankEvalResponse. Closes #19889 --- .../index/rankeval/RankEvalResponse.java | 49 +++++++++++-------- .../rankeval/TransportRankEvalAction.java | 37 +++++++------- .../index/rankeval/RankEvalRequestIT.java | 10 ++-- .../index/rankeval/RankEvalResponseTests.java | 13 +++-- .../test/rank_eval/30_failures.yaml | 38 ++++++++++++++ 5 files changed, 103 insertions(+), 44 deletions(-) create mode 100644 modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yaml diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 81994e86f24..23c4e27fb8c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -29,7 +30,6 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.Objects; /** * For each qa specification identified by its id this response returns the respective @@ -47,13 +47,16 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { private double qualityLevel; /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ private Map details; + /**Mapping from intent id to potential exceptions that were thrown on query execution.*/ + private Map failures; public RankEvalResponse() { } - public RankEvalResponse(double qualityLevel, Map partialResults) { + public RankEvalResponse(double qualityLevel, Map partialResults, Map failures) { this.qualityLevel = qualityLevel; this.details = partialResults; + this.failures = failures; } public double getQualityLevel() { @@ -64,9 +67,13 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { return Collections.unmodifiableMap(details); } + public Map getFailures() { + return Collections.unmodifiableMap(failures); + } + @Override public String toString() { - return "RankEvalResponse, quality: " + qualityLevel + ", partial results: " + details; + return "RankEvalResponse, quality: " + qualityLevel + ", partial results: " + details + ", number of failures: " + failures.size(); } @Override @@ -78,6 +85,11 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { out.writeString(queryId); details.get(queryId).writeTo(out); } + out.writeVInt(failures.size()); + for (String queryId : failures.keySet()) { + out.writeString(queryId); + out.writeException(failures.get(queryId)); + } } @Override @@ -91,6 +103,12 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { EvalQueryQuality partial = new EvalQueryQuality(in); this.details.put(queryId, partial); } + int failuresSize = in.readVInt(); + this.failures = new HashMap<>(failuresSize); + for (int i = 0; i < failuresSize; i++) { + String queryId = in.readString(); + this.failures.put(queryId, in.readException()); + } } @Override @@ -102,25 +120,14 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { details.get(key).toXContent(builder, params); } builder.endObject(); + builder.startObject("failures"); + for (String key : failures.keySet()) { + builder.startObject(key); + ElasticsearchException.renderException(builder, params, failures.get(key)); + builder.endObject(); + } + builder.endObject(); builder.endObject(); return builder; } - - @Override - public final boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - RankEvalResponse other = (RankEvalResponse) obj; - return Objects.equals(qualityLevel, other.qualityLevel) && - Objects.equals(details, other.details); - } - - @Override - public final int hashCode() { - return Objects.hash(qualityLevel, details); - } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index d632534776f..9577fde5f31 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -64,14 +64,14 @@ public class TransportRankEvalAction extends HandledTransportAction listener) { RankEvalSpec qualityTask = request.getRankEvalSpec(); - Map> unknownDocs = new ConcurrentHashMap<>(); Collection specifications = qualityTask.getSpecifications(); AtomicInteger responseCounter = new AtomicInteger(specifications.size()); Map partialResults = new ConcurrentHashMap<>(specifications.size()); + Map errors = new ConcurrentHashMap<>(specifications.size()); for (RatedRequest querySpecification : specifications) { - final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask, querySpecification, - partialResults, unknownDocs, responseCounter); + final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask.getMetric(), querySpecification, + partialResults, errors, responseCounter); SearchSourceBuilder specRequest = querySpecification.getTestRequest(); List summaryFields = querySpecification.getSummaryFields(); if (summaryFields.isEmpty()) { @@ -95,14 +95,15 @@ public class TransportRankEvalAction extends HandledTransportAction listener; private RatedRequest specification; private Map requestDetails; - private RankEvalSpec task; + private Map errors; + private RankedListQualityMetric metric; private AtomicInteger responseCounter; - public RankEvalActionListener(ActionListener listener, RankEvalSpec task, RatedRequest specification, - Map details, Map> unknownDocs, - AtomicInteger responseCounter) { + public RankEvalActionListener(ActionListener listener, RankedListQualityMetric metric, RatedRequest specification, + Map details, Map errors, AtomicInteger responseCounter) { this.listener = listener; - this.task = task; + this.metric = metric; + this.errors = errors; this.specification = specification; this.requestDetails = details; this.responseCounter = responseCounter; @@ -111,20 +112,22 @@ public class TransportRankEvalAction extends HandledTransportAction indices = Arrays.asList(new String[] { "test" }); @@ -146,7 +146,11 @@ public class RankEvalRequestIT extends ESIntegTestCase { RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); - expectThrows(SearchPhaseExecutionException.class, () -> client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet()); + RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + assertEquals(1, response.getFailures().size()); + ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(response.getFailures().get("broken_query")); + assertEquals("[range] time_zone can not be applied to non date field [text]", rootCauses[0].getMessage()); + } private static List createRelevant(String... docs) { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index ca524ad278b..fb24043dca1 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -48,7 +48,12 @@ public class RankEvalResponseTests extends ESTestCase { EvalQueryQuality evalQuality = new EvalQueryQuality(id, randomDoubleBetween(0.0, 1.0, true)); partials.put(id, evalQuality); } - return new RankEvalResponse(randomDouble(), partials); + int numberOfErrors = randomIntBetween(0, 2); + Map errors = new HashMap<>(numberOfRequests); + for (int i = 0; i < numberOfErrors; i++) { + errors.put(randomAsciiOfLengthBetween(3, 10), new IllegalArgumentException(randomAsciiOfLength(10))); + } + return new RankEvalResponse(randomDouble(), partials, errors); } public void testSerialization() throws IOException { @@ -58,8 +63,9 @@ public class RankEvalResponseTests extends ESTestCase { try (StreamInput in = output.bytes().streamInput()) { RankEvalResponse deserializedResponse = new RankEvalResponse(); deserializedResponse.readFrom(in); - assertEquals(randomResponse, deserializedResponse); - assertEquals(randomResponse.hashCode(), deserializedResponse.hashCode()); + assertEquals(randomResponse.getQualityLevel(), deserializedResponse.getQualityLevel(), Double.MIN_VALUE); + assertEquals(randomResponse.getPartialResults(), deserializedResponse.getPartialResults()); + assertEquals(randomResponse.getFailures().keySet(), deserializedResponse.getFailures().keySet()); assertNotSame(randomResponse, deserializedResponse); assertEquals(-1, in.read()); } @@ -75,5 +81,6 @@ public class RankEvalResponseTests extends ESTestCase { builder.startObject(); randomResponse.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); + // TODO check the correctness of the output } } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yaml new file mode 100644 index 00000000000..1d19a94cdac --- /dev/null +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yaml @@ -0,0 +1,38 @@ +--- +"Response format": + + - do: + index: + index: foo + type: bar + id: doc1 + body: { "bar": 1 } + + - do: + indices.refresh: {} + + - do: + rank_eval: + body: { + "requests" : [ + { + "id": "amsterdam_query", + "request": { "query": { "match_all" : { }}}, + "ratings": [ + {"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] + }, + { + "id" : "invalid_query", + "request": { "query": { "range" : { "bar" : { "time_zone": "+01:00" }}}}, + "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] + } + ], + "metric" : { "precision": { "ignore_unlabeled" : true }} + } + + - match: { rank_eval.quality_level: 1} + - match: { rank_eval.details.amsterdam_query.quality_level: 1.0} + - match: { rank_eval.details.amsterdam_query.unknown_docs: [ ]} + - match: { rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}} + + - is_true: rank_eval.failures.invalid_query From e0b15eafb01fb7865578517fbf239ee573546265 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Tue, 6 Dec 2016 10:30:25 +0100 Subject: [PATCH 082/297] Move rank-eval template compilation down to TransportRankEvalAction (#21855) Move rank-eval template compilation down to TransportRankEvalAction Closes #21777 and #21465 Search templates for rank_eval endpoint so far only worked when sent through REST end point However we also allow templates to be set through a Java API call to "setTemplate" on that same spec. This doesn't go through template execution so fails further down the line. To make this work, moved template execution further down, probably to TransportRankEvalAction. * Add template IT test for Java API * Move template compilation to TransportRankEvalAction --- .../index/rankeval/RankEvalSpec.java | 25 +--- .../index/rankeval/RatedRequest.java | 9 +- .../rankeval/TransportRankEvalAction.java | 34 ++++- .../build.gradle | 4 +- .../smoketest/SmokeMultipleTemplatesIT.java | 122 ++++++++++++++++++ 5 files changed, 163 insertions(+), 31 deletions(-) create mode 100644 qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index bae54425017..d8fd26c1152 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -23,23 +23,17 @@ import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Map; import java.util.Objects; /** @@ -154,24 +148,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } public static RankEvalSpec parse(XContentParser parser, RankEvalContext context, boolean templated) throws IOException { - RankEvalSpec spec = PARSER.parse(parser, context); - - if (templated) { - for (RatedRequest query_spec : spec.getSpecifications()) { - Map params = query_spec.getParams(); - Script scriptWithParams = new Script(spec.template.getType(), spec.template.getLang(), spec.template.getIdOrCode(), params); - String resolvedRequest = ((BytesReference) (context.getScriptService() - .executable(scriptWithParams, ScriptContext.Standard.SEARCH).run())).utf8ToString(); - try (XContentParser subParser = XContentFactory.xContent(resolvedRequest).createParser(resolvedRequest)) { - QueryParseContext parseContext = new QueryParseContext(context.getSearchRequestParsers().queryParsers, subParser, - context.getParseFieldMatcher()); - SearchSourceBuilder templateResult = SearchSourceBuilder.fromXContent(parseContext, context.getAggs(), - context.getSuggesters(), context.getSearchExtParsers()); - query_spec.setTestRequest(templateResult); - } - } - } - return spec; + return PARSER.parse(parser, context); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 6dc53244a6d..2b05e1a4976 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -74,7 +74,8 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { public RatedRequest(StreamInput in) throws IOException { this.specId = in.readString(); - testRequest = new SearchSourceBuilder(in); + testRequest = in.readOptionalWriteable(SearchSourceBuilder::new); + int indicesSize = in.readInt(); indices = new ArrayList<>(indicesSize); for (int i = 0; i < indicesSize; i++) { @@ -101,7 +102,8 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(specId); - testRequest.writeTo(out); + out.writeOptionalWriteable(testRequest); + out.writeInt(indices.size()); for (String index : indices) { out.writeString(index); @@ -255,8 +257,9 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(ID_FIELD.getPreferredName(), this.specId); - if (testRequest != null) + if (testRequest != null) { builder.field(REQUEST_FIELD.getPreferredName(), this.testRequest); + } builder.startObject(PARAMS_FIELD.getPreferredName()); for (Entry entry : this.params.entrySet()) { builder.field(entry.getKey(), entry.getValue()); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 9577fde5f31..7d73cf5d9ca 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -26,14 +26,24 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -51,12 +61,17 @@ import java.util.concurrent.atomic.AtomicInteger; * */ public class TransportRankEvalAction extends HandledTransportAction { private Client client; - + private ScriptService scriptService; + private SearchRequestParsers searchRequestParsers; + @Inject public TransportRankEvalAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, Client client, TransportService transportService) { + IndexNameExpressionResolver indexNameExpressionResolver, Client client, TransportService transportService, + SearchRequestParsers searchRequestParsers, ScriptService scriptService) { super(settings, RankEvalAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, RankEvalRequest::new); + this.searchRequestParsers = searchRequestParsers; + this.scriptService = scriptService; this.client = client; } @@ -69,10 +84,25 @@ public class TransportRankEvalAction extends HandledTransportAction partialResults = new ConcurrentHashMap<>(specifications.size()); Map errors = new ConcurrentHashMap<>(specifications.size()); + CompiledScript scriptWithoutParams = null; + if (qualityTask.getTemplate() != null) { + scriptWithoutParams = scriptService.compile(qualityTask.getTemplate(), ScriptContext.Standard.SEARCH, new HashMap<>()); + } for (RatedRequest querySpecification : specifications) { final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask.getMetric(), querySpecification, partialResults, errors, responseCounter); SearchSourceBuilder specRequest = querySpecification.getTestRequest(); + if (specRequest == null) { + Map params = querySpecification.getParams(); + String resolvedRequest = ((BytesReference) (scriptService.executable(scriptWithoutParams, params).run())).utf8ToString(); + try (XContentParser subParser = XContentFactory.xContent(resolvedRequest).createParser(resolvedRequest)) { + QueryParseContext parseContext = new QueryParseContext(searchRequestParsers.queryParsers, subParser, parseFieldMatcher); + specRequest = SearchSourceBuilder.fromXContent(parseContext, searchRequestParsers.aggParsers, + searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); + } catch (IOException e) { + listener.onFailure(e); + } + } List summaryFields = querySpecification.getSummaryFields(); if (summaryFields.isEmpty()) { specRequest.fetchSource(false); diff --git a/qa/smoke-test-rank-eval-with-mustache/build.gradle b/qa/smoke-test-rank-eval-with-mustache/build.gradle index 4860d5469af..4fdbaa04502 100644 --- a/qa/smoke-test-rank-eval-with-mustache/build.gradle +++ b/qa/smoke-test-rank-eval-with-mustache/build.gradle @@ -19,9 +19,9 @@ apply plugin: 'elasticsearch.rest-test' -/* + dependencies { testCompile project(path: ':modules:rank-eval', configuration: 'runtime') testCompile project(path: ':modules:lang-mustache', configuration: 'runtime') } -*/ + diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java new file mode 100644 index 00000000000..02213642d39 --- /dev/null +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java @@ -0,0 +1,122 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.smoketest; + +import org.elasticsearch.index.rankeval.Precision; +import org.elasticsearch.index.rankeval.RankEvalAction; +import org.elasticsearch.index.rankeval.RankEvalPlugin; +import org.elasticsearch.index.rankeval.RankEvalRequest; +import org.elasticsearch.index.rankeval.RankEvalRequestBuilder; +import org.elasticsearch.index.rankeval.RankEvalResponse; +import org.elasticsearch.index.rankeval.RankEvalSpec; +import org.elasticsearch.index.rankeval.RatedDocument; +import org.elasticsearch.index.rankeval.RatedRequest; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.test.ESIntegTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +public class SmokeMultipleTemplatesIT extends ESIntegTestCase { + + @Override + protected Collection> transportClientPlugins() { + return Arrays.asList(RankEvalPlugin.class); + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(RankEvalPlugin.class); + } + + @Before + public void setup() { + createIndex("test"); + ensureGreen(); + + client().prepareIndex("test", "testtype").setId("1") + .setSource("text", "berlin", "title", "Berlin, Germany").get(); + client().prepareIndex("test", "testtype").setId("2") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("3") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("4") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("5") + .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("6") + .setSource("text", "amsterdam").get(); + refresh(); + } + + public void testPrecisionAtRequest() throws IOException { + List indices = Arrays.asList(new String[] { "test" }); + List types = Arrays.asList(new String[] { "testtype" }); + + List specifications = new ArrayList<>(); + RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", null, indices, types, createRelevant("2", "3", "4", "5")); + Map ams_params = new HashMap<>(); + ams_params.put("querystring", "amsterdam"); + amsterdamRequest.setParams(ams_params); + specifications.add(amsterdamRequest); + + RatedRequest berlinRequest = new RatedRequest("berlin_query", null, indices, types, createRelevant("1")); + Map berlin_params = new HashMap<>(); + berlin_params.put("querystring", "berlin"); + berlinRequest.setParams(berlin_params); + specifications.add(berlinRequest); + + Precision metric = new Precision(); + RankEvalSpec task = new RankEvalSpec(specifications, metric); + task.setTemplate( + new Script( + ScriptType.INLINE, + "mustache", "{\"query\": {\"match\": {\"text\": \"{{querystring}}\"}}}", + new HashMap<>())); + + RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); + builder.setRankEvalSpec(task); + + RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + assertEquals(0.9, response.getQualityLevel(), Double.MIN_VALUE); + } + + private static List createRelevant(String... docs) { + List relevant = new ArrayList<>(); + for (String doc : docs) { + relevant.add(new RatedDocument("test", "testtype", doc, Rating.RELEVANT.ordinal())); + } + return relevant; + } + + public enum Rating { + IRRELEVANT, RELEVANT; + } + + } From 165cec275743cc3a30059b5b37e68a0725afcff8 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 7 Dec 2016 11:47:47 +0100 Subject: [PATCH 083/297] Serialisation and validation checks for rank evaluation request components (#21975) Adds tests around serialisation/validation checks for rank evaluation request components * Add null/ empty string checks to RatedDocument constructor * Add mutation test to RatedDocument serialization tests. * Reorganise rank-eval RatedDocument tests and add serialisation test. * Add roundtrip serialisation testing for RatedRequests * Adds serialisation testing and equals/hashcode testing for RatedRequest. * Fixes a bug in previous equals implementation of RatedRequest along the way. * Add roundtrip tests for Precision and ReciprocalRank * Also fixes a bug with serialising ReciprocalRank. * Add roundtrip testing for DiscountedCumulativeGain * Add serialisation test for DocumentKey and fix test init * Add check that relevant doc threshold is always positive for precision. * Check that relevant threshold is always positive for precision and reciprocal rank Closes #21401 --- .../index/rankeval/DocumentKey.java | 11 +++ .../index/rankeval/Precision.java | 3 + .../index/rankeval/RatedRequest.java | 4 +- .../index/rankeval/ReciprocalRank.java | 6 +- .../DiscountedCumulativeGainTests.java | 28 ++++++ .../index/rankeval/DocumentKeyTests.java | 18 +++- .../index/rankeval/PrecisionTests.java | 33 +++++++ .../index/rankeval/RatedDocumentTests.java | 48 ++++++++++ .../index/rankeval/RatedRequestsTests.java | 88 +++++++++++++++++++ .../index/rankeval/ReciprocalRankTests.java | 36 +++++++- 10 files changed, 267 insertions(+), 8 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java index 55983d11e49..44b0aa21359 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -47,6 +48,16 @@ public class DocumentKey extends ToXContentToBytes implements Writeable { } public DocumentKey(String index, String type, String docId) { + if (Strings.isNullOrEmpty(index)) { + throw new IllegalArgumentException("Index must be set for each rated document"); + } + if(Strings.isNullOrEmpty(type)) { + throw new IllegalArgumentException("Type must be set for each rated document"); + } + if (Strings.isNullOrEmpty(docId)) { + throw new IllegalArgumentException("DocId must be set for each rated document"); + } + this.index = index; this.type = type; this.docId = docId; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java index f85e9460120..1536388391b 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java @@ -90,6 +90,9 @@ public class Precision implements RankedListQualityMetric { * Sets the rating threshold above which ratings are considered to be "relevant" for this metric. * */ public void setRelevantRatingThreshhold(int threshold) { + if (threshold < 0) { + throw new IllegalArgumentException("Relevant rating threshold for precision must be positive integer."); + } this.relevantRatingThreshhold = threshold; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 2b05e1a4976..3d334193ec8 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -289,12 +289,14 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { if (obj == null || getClass() != obj.getClass()) { return false; } + RatedRequest other = (RatedRequest) obj; + return Objects.equals(specId, other.specId) && Objects.equals(testRequest, other.testRequest) && Objects.equals(indices, other.indices) && Objects.equals(types, other.types) && - Objects.equals(summaryFields, summaryFields) && + Objects.equals(summaryFields, other.summaryFields) && Objects.equals(ratedDocs, other.ratedDocs) && Objects.equals(params, other.params); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 58cc8a3f100..a0238717169 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -57,7 +57,7 @@ public class ReciprocalRank implements RankedListQualityMetric { } public ReciprocalRank(StreamInput in) throws IOException { - this.relevantRatingThreshhold = in.readInt(); + this.relevantRatingThreshhold = in.readVInt(); } @Override @@ -69,6 +69,10 @@ public class ReciprocalRank implements RankedListQualityMetric { * Sets the rating threshold above which ratings are considered to be "relevant" for this metric. * */ public void setRelevantRatingThreshhold(int threshold) { + if (threshold < 0) { + throw new IllegalArgumentException("Relevant rating threshold for precision must be positive integer."); + } + this.relevantRatingThreshhold = threshold; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 32a4d0dd82a..80f954f3914 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -214,4 +214,32 @@ public class DiscountedCumulativeGainTests extends ESTestCase { assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); } + + public void testSerialization() throws IOException { + DiscountedCumulativeGain original = createTestItem(); + DiscountedCumulativeGain deserialized = RankEvalTestHelper.copy(original, DiscountedCumulativeGain::new); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + + public void testEqualsAndHash() throws IOException { + DiscountedCumulativeGain testItem = createTestItem(); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), + RankEvalTestHelper.copy(testItem, DiscountedCumulativeGain::new)); + } + + private DiscountedCumulativeGain mutateTestItem(DiscountedCumulativeGain original) { + boolean normalise = original.getNormalize(); + int unknownDocRating = original.getUnknownDocRating(); + DiscountedCumulativeGain gain = new DiscountedCumulativeGain(); + gain.setNormalize(normalise); + gain.setUnknownDocRating(unknownDocRating); + + List mutators = new ArrayList<>(); + mutators.add(() -> gain.setNormalize(! original.getNormalize())); + mutators.add(() -> gain.setUnknownDocRating(randomValueOtherThan(unknownDocRating, () -> randomIntBetween(0, 10)))); + randomFrom(mutators).run(); + return gain; + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java index 3cbdc4dc274..992f3f49fb1 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java @@ -27,13 +27,13 @@ import java.io.IOException; public class DocumentKeyTests extends ESTestCase { static DocumentKey createRandomRatedDocumentKey() { - String index = randomAsciiOfLengthBetween(0, 10); - String type = randomAsciiOfLengthBetween(0, 10); - String docId = randomAsciiOfLengthBetween(0, 10); + String index = randomAsciiOfLengthBetween(1, 10); + String type = randomAsciiOfLengthBetween(1, 10); + String docId = randomAsciiOfLengthBetween(1, 10); return new DocumentKey(index, type, docId); } - public DocumentKey createRandomTestItem() { + public DocumentKey createTestItem() { return createRandomRatedDocumentKey(); } @@ -62,4 +62,14 @@ public class DocumentKeyTests extends ESTestCase { RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), new DocumentKey(testItem.getIndex(), testItem.getType(), testItem.getDocID())); } + + public void testSerialization() throws IOException { + DocumentKey original = createTestItem(); + DocumentKey deserialized = RankEvalTestHelper.copy(original, DocumentKey::new); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + + } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java index 75c7818f40e..eec41ea2fec 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java @@ -167,6 +167,11 @@ public class PrecisionTests extends ESTestCase { assertEquals(0.3, metric.combine(partialResults), Double.MIN_VALUE); } + public void testInvalidRelevantThreshold() { + Precision prez = new Precision(); + expectThrows(IllegalArgumentException.class, () -> prez.setRelevantRatingThreshhold(-1)); + } + public static Precision createTestItem() { Precision precision = new Precision(); if (randomBoolean()) { @@ -186,6 +191,34 @@ public class PrecisionTests extends ESTestCase { assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); } + + public void testSerialization() throws IOException { + Precision original = createTestItem(); + Precision deserialized = RankEvalTestHelper.copy(original, Precision::new); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + + public void testEqualsAndHash() throws IOException { + Precision testItem = createTestItem(); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), + RankEvalTestHelper.copy(testItem, Precision::new)); + } + + private Precision mutateTestItem(Precision original) { + boolean ignoreUnlabeled = original.getIgnoreUnlabeled(); + int relevantThreshold = original.getRelevantRatingThreshold(); + Precision precision = new Precision(); + precision.setIgnoreUnlabeled(ignoreUnlabeled); + precision.setRelevantRatingThreshhold(relevantThreshold); + + List mutators = new ArrayList<>(); + mutators.add(() -> precision.setIgnoreUnlabeled(! ignoreUnlabeled)); + mutators.add(() -> precision.setRelevantRatingThreshhold(randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10)))); + randomFrom(mutators).run(); + return precision; + } private static SearchHit[] toSearchHits(List rated, String index, String type) { InternalSearchHit[] hits = new InternalSearchHit[rated.size()]; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index 33266b897f7..f42c6e3e542 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -44,4 +44,52 @@ public class RatedDocumentTests extends ESTestCase { assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); } + + public void testSerialization() throws IOException { + RatedDocument original = createRatedDocument(); + RatedDocument deserialized = RankEvalTestHelper.copy(original, RatedDocument::new); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + + public void testEqualsAndHash() throws IOException { + RatedDocument testItem = createRatedDocument(); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), + RankEvalTestHelper.copy(testItem, RatedDocument::new)); + } + + public void testInvalidParsing() throws IOException { + expectThrows(IllegalArgumentException.class, () -> new RatedDocument(null, "abc", "abc", 10)); + expectThrows(IllegalArgumentException.class, () -> new RatedDocument("", "abc", "abc", 10)); + expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", null, "abc", 10)); + expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "", "abc", 10)); + expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "abc", null, 10)); + expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "abc", "", 10)); + } + + private static RatedDocument mutateTestItem(RatedDocument original) { + int rating = original.getRating(); + String index = original.getIndex(); + String type = original.getType(); + String docId = original.getDocID(); + + switch (randomIntBetween(0, 3)) { + case 0: + rating = randomValueOtherThan(rating, () -> randomInt()); + break; + case 1: + index = randomValueOtherThan(index, () -> randomAsciiOfLength(10)); + break; + case 2: + type = randomValueOtherThan(type, () -> randomAsciiOfLength(10)); + break; + case 3: + docId = randomValueOtherThan(docId, () -> randomAsciiOfLength(10)); + break; + default: + throw new IllegalStateException("The test should only allow two parameters mutated"); + } + return new RatedDocument(index, type, docId, rating); + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index acd0c1ccb3e..e43b0e041d0 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -20,11 +20,13 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ParseFieldRegistry; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.SearchModule; @@ -131,6 +133,92 @@ public class RatedRequestsTests extends ESTestCase { assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); } + + public void testSerialization() throws IOException { + List indices = new ArrayList<>(); + int size = randomIntBetween(0, 20); + for (int i = 0; i < size; i++) { + indices.add(randomAsciiOfLengthBetween(0, 50)); + } + + List types = new ArrayList<>(); + size = randomIntBetween(0, 20); + for (int i = 0; i < size; i++) { + types.add(randomAsciiOfLengthBetween(0, 50)); + } + + RatedRequest original = createTestItem(indices, types); + + List namedWriteables = new ArrayList<>(); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + + RatedRequest deserialized = RankEvalTestHelper.copy(original, RatedRequest::new, new NamedWriteableRegistry(namedWriteables)); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + + public void testEqualsAndHash() throws IOException { + List indices = new ArrayList<>(); + int size = randomIntBetween(0, 20); + for (int i = 0; i < size; i++) { + indices.add(randomAsciiOfLengthBetween(0, 50)); + } + + List types = new ArrayList<>(); + size = randomIntBetween(0, 20); + for (int i = 0; i < size; i++) { + types.add(randomAsciiOfLengthBetween(0, 50)); + } + + RatedRequest testItem = createTestItem(indices, types); + + List namedWriteables = new ArrayList<>(); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), + RankEvalTestHelper.copy(testItem, RatedRequest::new, new NamedWriteableRegistry(namedWriteables))); + } + + private RatedRequest mutateTestItem(RatedRequest original) { + String specId = original.getSpecId(); + int size = original.getTestRequest().size(); + List ratedDocs = original.getRatedDocs(); + List indices = original.getIndices(); + List types = original.getTypes(); + Map params = original.getParams(); + List summaryFields = original.getSummaryFields(); + + SearchSourceBuilder testRequest = new SearchSourceBuilder(); + testRequest.size(size); + testRequest.query(new MatchAllQueryBuilder()); + + RatedRequest ratedRequest = new RatedRequest(specId, testRequest, indices, types, ratedDocs); + ratedRequest.setIndices(indices); + ratedRequest.setTypes(types); + ratedRequest.setParams(params); + ratedRequest.setSummaryFields(summaryFields); + + List mutators = new ArrayList<>(); + mutators.add(() -> ratedRequest.setSpecId(randomValueOtherThan(specId, () -> randomAsciiOfLength(10)))); + mutators.add(() -> ratedRequest.getTestRequest().size(randomValueOtherThan(size, () -> randomInt()))); + mutators.add(() -> ratedRequest.setRatedDocs( + Arrays.asList(randomValueOtherThanMany(ratedDocs::contains, () -> RatedDocumentTests.createRatedDocument())))); + mutators.add(() -> ratedRequest.setIndices( + Arrays.asList(randomValueOtherThanMany(indices::contains, () -> randomAsciiOfLength(10))))); + + HashMap modified = new HashMap<>(); + modified.putAll(params); + modified.put("one_more_key", "one_more_value"); + mutators.add(() -> ratedRequest.setParams(modified)); + + mutators.add(() -> ratedRequest.setSummaryFields( + Arrays.asList(randomValueOtherThanMany(summaryFields::contains, () -> randomAsciiOfLength(10))))); + + + randomFrom(mutators).run(); + return ratedRequest; + } public void testDuplicateRatedDocThrowsException() { RatedRequest request = createTestItem(Arrays.asList("index"), Arrays.asList("type")); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 4ce34ebc0d1..b3c2e0d7cdc 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -123,8 +123,7 @@ public class ReciprocalRankTests extends ESTestCase { } public void testXContentRoundtrip() throws IOException { - ReciprocalRank testItem = new ReciprocalRank(); - testItem.setRelevantRatingThreshhold(randomIntBetween(0, 20)); + ReciprocalRank testItem = createTestItem(); XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); @@ -146,4 +145,37 @@ public class ReciprocalRankTests extends ESTestCase { } return hits; } + + private ReciprocalRank createTestItem() { + ReciprocalRank testItem = new ReciprocalRank(); + testItem.setRelevantRatingThreshhold(randomIntBetween(0, 20)); + return testItem; + } + + public void testSerialization() throws IOException { + ReciprocalRank original = createTestItem(); + + ReciprocalRank deserialized = RankEvalTestHelper.copy(original, ReciprocalRank::new); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + + public void testEqualsAndHash() throws IOException { + ReciprocalRank testItem = createTestItem(); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), + RankEvalTestHelper.copy(testItem, ReciprocalRank::new)); + } + + private ReciprocalRank mutateTestItem(ReciprocalRank testItem) { + int relevantThreshold = testItem.getRelevantRatingThreshold(); + ReciprocalRank rank = new ReciprocalRank(); + rank.setRelevantRatingThreshhold(randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10))); + return rank; + } + + public void testInvalidRelevantThreshold() { + ReciprocalRank prez = new ReciprocalRank(); + expectThrows(IllegalArgumentException.class, () -> prez.setRelevantRatingThreshhold(-1)); + } } From 6d1a658106e74f82c4c20b9c648dd7f78c216c24 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Tue, 13 Dec 2016 11:16:53 +0100 Subject: [PATCH 084/297] Fix compile issues after merging in master. --- .../index/rankeval/RestRankEvalAction.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index ce06d794276..36d260b2efa 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -187,14 +187,11 @@ public class RestRankEvalAction extends BaseRestHandler { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { RankEvalRequest rankEvalRequest = new RankEvalRequest(); - BytesReference restContent = RestActions.hasBodyContent(request) ? RestActions.getRestContent(request) : null; - try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { + try (XContentParser parser = request.contentOrSourceParamParser()) { QueryParseContext parseContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher); - if (restContent != null) { - parseRankEvalRequest(rankEvalRequest, request, - // TODO can we get rid of aggregators parsers and suggesters? - new RankEvalContext(parseFieldMatcher, parseContext, searchRequestParsers, scriptService)); - } + // TODO can we get rid of aggregators parsers and suggesters? + parseRankEvalRequest(rankEvalRequest, request, + new RankEvalContext(parseFieldMatcher, parseContext, searchRequestParsers, scriptService)); } return channel -> client.executeLocally(RankEvalAction.INSTANCE, rankEvalRequest, new RestToXContentListener(channel)); From 58342d4c9a2846866da970cf9e8f9065e73cf4ef Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Tue, 13 Dec 2016 11:21:57 +0100 Subject: [PATCH 085/297] Add checks to RankEvalSpec to safe guard against missing parameters. (#22026) Add checks to RankEvalSpec to safe guard against missing parameters. Fail early in case no metric is supplied, no rated requests are supplied or the search source builder is missing but no template is supplied neither. Add stricter checks around rank eval request parsing: Fail if in a rated request we see both, a verbatim request as well as request template parameters. Relates to #21260 --- .../index/rankeval/RankEvalSpec.java | 86 ++++++----- .../index/rankeval/RatedRequest.java | 141 +++++++++--------- .../index/rankeval/RestRankEvalAction.java | 12 +- .../rankeval/TransportRankEvalAction.java | 38 ++--- .../index/rankeval/RankEvalRequestIT.java | 23 ++- .../index/rankeval/RankEvalSpecTests.java | 74 ++++++--- .../index/rankeval/RatedRequestsTests.java | 136 ++++++++++++----- .../smoketest/SmokeMultipleTemplatesIT.java | 19 ++- .../test/rank-eval/30_template.yaml | 2 +- .../rest-api-spec/api/rank_eval_template.json | 25 ---- 10 files changed, 322 insertions(+), 234 deletions(-) delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval_template.json diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index d8fd26c1152..4626b604f71 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -26,7 +26,7 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.script.Script; @@ -52,13 +52,31 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { @Nullable private Script template; - public RankEvalSpec() { - // TODO think if no args ctor is okay + public RankEvalSpec(Collection ratedRequests, RankedListQualityMetric metric, Script template) { + if (ratedRequests == null || ratedRequests.size() < 1) { + throw new IllegalStateException( + "Cannot evaluate ranking if no search requests with rated results are provided. Seen: " + ratedRequests); + } + if (metric == null) { + throw new IllegalStateException( + "Cannot evaluate ranking if no evaluation metric is provided."); + } + if (template == null) { + for (RatedRequest request : ratedRequests) { + if (request.getTestRequest() == null) { + throw new IllegalStateException( + "Cannot evaluate ranking if neither template nor test request is provided. Seen for request id: " + + request.getId()); + } + } + } + this.ratedRequests = ratedRequests; + this.metric = metric; + this.template = template; } - public RankEvalSpec(Collection specs, RankedListQualityMetric metric) { - this.ratedRequests = specs; - this.metric = metric; + public RankEvalSpec(Collection ratedRequests, RankedListQualityMetric metric) { + this(ratedRequests, metric, null); } public RankEvalSpec(StreamInput in) throws IOException { @@ -88,31 +106,16 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } } - /** Set the metric to use for quality evaluation. */ - public void setMetric(RankedListQualityMetric metric) { - this.metric = metric; - } - /** Returns the metric to use for quality evaluation.*/ public RankedListQualityMetric getMetric() { return metric; } /** Returns a list of intent to query translation specifications to evaluate. */ - public Collection getSpecifications() { + public Collection getRatedRequests() { return ratedRequests; } - /** Set the list of intent to query translation specifications to evaluate. */ - public void setSpecifications(Collection specifications) { - this.ratedRequests = specifications; - } - - /** Set the template to base test requests on. */ - public void setTemplate(Script script) { - this.template = script; - } - /** Returns the template to base test requests on. */ public Script getTemplate() { return this.template; @@ -121,34 +124,37 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { private static final ParseField TEMPLATE_FIELD = new ParseField("template"); private static final ParseField METRIC_FIELD = new ParseField("metric"); private static final ParseField REQUESTS_FIELD = new ParseField("requests"); - private static final ObjectParser PARSER = new ObjectParser<>("rank_eval", RankEvalSpec::new); + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("rank_eval", + a -> new RankEvalSpec((Collection) a[0], (RankedListQualityMetric) a[1], (Script) a[2])); static { - PARSER.declareObject(RankEvalSpec::setMetric, (p, c) -> { - try { - return RankedListQualityMetric.fromXContent(p, c); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); - } - } , METRIC_FIELD); - PARSER.declareObject(RankEvalSpec::setTemplate, (p, c) -> { - try { - return Script.parse(p, c.getParseFieldMatcher(), "mustache"); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); - } - }, TEMPLATE_FIELD); - PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> { + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> { try { return RatedRequest.fromXContent(p, c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } } , REQUESTS_FIELD); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { + try { + return RankedListQualityMetric.fromXContent(p, c); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + } + } , METRIC_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { + try { + return Script.parse(p, c.getParseFieldMatcher(), "mustache"); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + } + }, TEMPLATE_FIELD); } - public static RankEvalSpec parse(XContentParser parser, RankEvalContext context, boolean templated) throws IOException { - return PARSER.parse(parser, context); + public static RankEvalSpec parse(XContentParser parser, RankEvalContext context) throws IOException { + return PARSER.apply(parser, context); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 3d334193ec8..6c61ade5c6a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -20,12 +20,13 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -48,32 +49,55 @@ import java.util.Set; * */ @SuppressWarnings("unchecked") public class RatedRequest extends ToXContentToBytes implements Writeable { - private String specId; - private SearchSourceBuilder testRequest; + private String id; private List indices = new ArrayList<>(); private List types = new ArrayList<>(); private List summaryFields = new ArrayList<>(); /** Collection of rated queries for this query QA specification.*/ private List ratedDocs = new ArrayList<>(); + /** Search request to execute for this rated request, can be null, if template and corresponding params are supplied. */ + @Nullable + private SearchSourceBuilder testRequest; /** Map of parameters to use for filling a query template, can be used instead of providing testRequest. */ private Map params = new HashMap<>(); - public RatedRequest() { - // ctor that doesn't require all args to be present immediatly is easier to use with ObjectParser - // TODO decide if we can require only id as mandatory, set default values for the rest? + public RatedRequest(String id, List ratedDocs, SearchSourceBuilder testRequest, Map params) { + if (params != null && (params.size() > 0 && testRequest != null)) { + throw new IllegalArgumentException( + "Ambiguous rated request: Set both, verbatim test request and test request template parameters."); + } + if ((params == null || params.size() < 1) && testRequest == null) { + throw new IllegalArgumentException( + "Need to set at least test request or test request template parameters."); + } + // No documents with same _index/_type/id allowed. + Set docKeys = new HashSet<>(); + for (RatedDocument doc : ratedDocs) { + if (docKeys.add(doc.getKey()) == false) { + String docKeyToString = doc.getKey().toString().replaceAll("\n", "").replaceAll(" ", " "); + throw new IllegalArgumentException( + "Found duplicate rated document key [" + docKeyToString + "]"); + } + } + + this.id = id; + this.testRequest = testRequest; + this.ratedDocs = ratedDocs; + if (params != null) { + this.params = params; + } + } + + public RatedRequest(String id, List ratedDocs, Map params) { + this(id, ratedDocs, null, params); } - public RatedRequest(String specId, SearchSourceBuilder testRequest, List indices, List types, - List ratedDocs) { - this.specId = specId; - this.testRequest = testRequest; - this.indices = indices; - this.types = types; - setRatedDocs(ratedDocs); + public RatedRequest(String id, List ratedDocs, SearchSourceBuilder testRequest) { + this(id, ratedDocs, testRequest, new HashMap<>()); } public RatedRequest(StreamInput in) throws IOException { - this.specId = in.readString(); + this.id = in.readString(); testRequest = in.readOptionalWriteable(SearchSourceBuilder::new); int indicesSize = in.readInt(); @@ -101,7 +125,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(specId); + out.writeString(id); out.writeOptionalWriteable(testRequest); out.writeInt(indices.size()); @@ -127,34 +151,25 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { return testRequest; } - public void setTestRequest(SearchSourceBuilder testRequest) { - this.testRequest = testRequest; + public void setIndices(List indices) { + this.indices = indices; } public List getIndices() { return indices; } - public void setIndices(List indices) { - this.indices = indices; + public void setTypes(List types) { + this.types = types; } public List getTypes() { return types; } - public void setTypes(List types) { - this.types = types; - } - /** Returns a user supplied spec id for easier referencing. */ - public String getSpecId() { - return specId; - } - - /** Sets a user supplied spec id for easier referencing. */ - public void setSpecId(String specId) { - this.specId = specId; + public String getId() { + return id; } /** Returns a list of rated documents to evaluate. */ @@ -162,70 +177,56 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { return ratedDocs; } - /** - * Set a list of rated documents for this query. - * No documents with same _index/_type/id allowed. - **/ - public void setRatedDocs(List ratedDocs) { - Set docKeys = new HashSet<>(); - for (RatedDocument doc : ratedDocs) { - if (docKeys.add(doc.getKey()) == false) { - String docKeyToString = doc.getKey().toString().replaceAll("\n", "").replaceAll(" ", " "); - throw new IllegalArgumentException( - "Found duplicate rated document key [" + docKeyToString + "]"); - } - } - this.ratedDocs = ratedDocs; - } - - public void setParams(Map params) { - this.params = params; - } - public Map getParams() { return this.params; } - public void setSummaryFields(List fields) { - this.summaryFields = fields; - } - /** Returns a list of fields that are included in the docs summary of matched documents. */ public List getSummaryFields() { return summaryFields; } + + public void setSummaryFields(List summaryFields) { + if (summaryFields == null) { + throw new IllegalArgumentException("Setting summaryFields to null not allowed."); + } + this.summaryFields = summaryFields; + } private static final ParseField ID_FIELD = new ParseField("id"); private static final ParseField REQUEST_FIELD = new ParseField("request"); private static final ParseField RATINGS_FIELD = new ParseField("ratings"); private static final ParseField PARAMS_FIELD = new ParseField("params"); private static final ParseField FIELDS_FIELD = new ParseField("summary_fields"); - private static final ObjectParser PARSER = new ObjectParser<>("requests", RatedRequest::new); + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("requests", a -> new RatedRequest( + (String) a[0], (List) a[1], (SearchSourceBuilder) a[2], (Map) a[3])); static { - PARSER.declareString(RatedRequest::setSpecId, ID_FIELD); - PARSER.declareStringArray(RatedRequest::setSummaryFields, FIELDS_FIELD); - PARSER.declareObject(RatedRequest::setTestRequest, (p, c) -> { - try { - return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters(), c.getSearchExtParsers()); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing request", ex); - } - } , REQUEST_FIELD); - PARSER.declareObjectArray(RatedRequest::setRatedDocs, (p, c) -> { + PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> { try { return RatedDocument.fromXContent(p, c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex); } }, RATINGS_FIELD); - PARSER.declareObject(RatedRequest::setParams, (p, c) -> { + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { + try { + return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters(), c.getSearchExtParsers()); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing request", ex); + } + } , REQUEST_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { try { return (Map) p.map(); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex); } }, PARAMS_FIELD); + PARSER.declareStringArray(RatedRequest::setSummaryFields, FIELDS_FIELD); } /** @@ -250,13 +251,13 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { * } */ public static RatedRequest fromXContent(XContentParser parser, RankEvalContext context) throws IOException { - return PARSER.parse(parser, context); + return PARSER.apply(parser, context); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(ID_FIELD.getPreferredName(), this.specId); + builder.field(ID_FIELD.getPreferredName(), this.id); if (testRequest != null) { builder.field(REQUEST_FIELD.getPreferredName(), this.testRequest); } @@ -292,7 +293,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { RatedRequest other = (RatedRequest) obj; - return Objects.equals(specId, other.specId) && + return Objects.equals(id, other.id) && Objects.equals(testRequest, other.testRequest) && Objects.equals(indices, other.indices) && Objects.equals(types, other.types) && @@ -303,6 +304,6 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { @Override public final int hashCode() { - return Objects.hash(specId, testRequest, indices, types, summaryFields, ratedDocs, params); + return Objects.hash(id, testRequest, indices, types, summaryFields, ratedDocs, params); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 36d260b2efa..fb042e1a8d6 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -175,13 +175,6 @@ public class RestRankEvalAction extends BaseRestHandler { controller.registerHandler(POST, "/{index}/_rank_eval", this); controller.registerHandler(GET, "/{index}/{type}/_rank_eval", this); controller.registerHandler(POST, "/{index}/{type}/_rank_eval", this); - - controller.registerHandler(GET, "/_rank_eval/template", this); - controller.registerHandler(POST, "/_rank_eval/template", this); - controller.registerHandler(GET, "/{index}/_rank_eval/template", this); - controller.registerHandler(POST, "/{index}/_rank_eval/template", this); - controller.registerHandler(GET, "/{index}/{type}/_rank_eval/template", this); - controller.registerHandler(POST, "/{index}/{type}/_rank_eval/template", this); } @Override @@ -202,9 +195,8 @@ public class RestRankEvalAction extends BaseRestHandler { List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); List types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type"))); RankEvalSpec spec = null; - boolean containsTemplate = request.path().contains("template"); - spec = RankEvalSpec.parse(context.parser(), context, containsTemplate); - for (RatedRequest specification : spec.getSpecifications()) { + spec = RankEvalSpec.parse(context.parser(), context); + for (RatedRequest specification : spec.getRatedRequests()) { specification.setIndices(indices); specification.setTypes(types); }; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 7d73cf5d9ca..9359e72c6e2 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -79,7 +79,7 @@ public class TransportRankEvalAction extends HandledTransportAction listener) { RankEvalSpec qualityTask = request.getRankEvalSpec(); - Collection specifications = qualityTask.getSpecifications(); + Collection specifications = qualityTask.getRatedRequests(); AtomicInteger responseCounter = new AtomicInteger(specifications.size()); Map partialResults = new ConcurrentHashMap<>(specifications.size()); Map errors = new ConcurrentHashMap<>(specifications.size()); @@ -88,34 +88,36 @@ public class TransportRankEvalAction extends HandledTransportAction()); } - for (RatedRequest querySpecification : specifications) { - final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask.getMetric(), querySpecification, + for (RatedRequest ratedRequest : specifications) { + final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask.getMetric(), ratedRequest, partialResults, errors, responseCounter); - SearchSourceBuilder specRequest = querySpecification.getTestRequest(); - if (specRequest == null) { - Map params = querySpecification.getParams(); + SearchSourceBuilder ratedSearchSource = ratedRequest.getTestRequest(); + if (ratedSearchSource == null) { + Map params = ratedRequest.getParams(); String resolvedRequest = ((BytesReference) (scriptService.executable(scriptWithoutParams, params).run())).utf8ToString(); try (XContentParser subParser = XContentFactory.xContent(resolvedRequest).createParser(resolvedRequest)) { QueryParseContext parseContext = new QueryParseContext(searchRequestParsers.queryParsers, subParser, parseFieldMatcher); - specRequest = SearchSourceBuilder.fromXContent(parseContext, searchRequestParsers.aggParsers, + ratedSearchSource = SearchSourceBuilder.fromXContent(parseContext, searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); } catch (IOException e) { listener.onFailure(e); } } - List summaryFields = querySpecification.getSummaryFields(); + List summaryFields = ratedRequest.getSummaryFields(); if (summaryFields.isEmpty()) { - specRequest.fetchSource(false); + ratedSearchSource.fetchSource(false); } else { - specRequest.fetchSource(summaryFields.toArray(new String[summaryFields.size()]), new String[0]); + ratedSearchSource.fetchSource(summaryFields.toArray(new String[summaryFields.size()]), new String[0]); } - String[] indices = new String[querySpecification.getIndices().size()]; - querySpecification.getIndices().toArray(indices); - SearchRequest templatedRequest = new SearchRequest(indices, specRequest); - String[] types = new String[querySpecification.getTypes().size()]; - querySpecification.getTypes().toArray(types); + String[] indices = new String[ratedRequest.getIndices().size()]; + indices = ratedRequest.getIndices().toArray(indices); + SearchRequest templatedRequest = new SearchRequest(indices, ratedSearchSource); + + String[] types = new String[ratedRequest.getTypes().size()]; + types = ratedRequest.getTypes().toArray(types); templatedRequest.types(types); + client.search(templatedRequest, searchListener); } } @@ -142,14 +144,14 @@ public class TransportRankEvalAction extends HandledTransportAction specifications = new ArrayList<>(); SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5")); + RatedRequest amsterdamRequest = new RatedRequest( + "amsterdam_query", createRelevant("2", "3", "4", "5"), testQuery); + amsterdamRequest.setIndices(indices); + amsterdamRequest.setTypes(types); amsterdamRequest.setSummaryFields(Arrays.asList(new String[]{ "text", "title" })); + specifications.add(amsterdamRequest); - RatedRequest berlinRequest = new RatedRequest("berlin_query", testQuery, indices, types, createRelevant("1")); + RatedRequest berlinRequest = new RatedRequest( + "berlin_query", createRelevant("1"), testQuery); + berlinRequest.setIndices(indices); + berlinRequest.setTypes(types); berlinRequest.setSummaryFields(Arrays.asList(new String[]{ "text", "title" })); + specifications.add(berlinRequest); Precision metric = new Precision(); @@ -135,11 +143,18 @@ public class RankEvalRequestIT extends ESIntegTestCase { List specifications = new ArrayList<>(); SearchSourceBuilder amsterdamQuery = new SearchSourceBuilder(); amsterdamQuery.query(new MatchAllQueryBuilder()); - specifications.add(new RatedRequest("amsterdam_query", amsterdamQuery, indices, types, createRelevant("2", "3", "4", "5"))); + RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", createRelevant("2", "3", "4", "5"), amsterdamQuery); + amsterdamRequest.setIndices(indices); + amsterdamRequest.setTypes(types); + specifications.add(amsterdamRequest); + SearchSourceBuilder brokenQuery = new SearchSourceBuilder(); RangeQueryBuilder brokenRangeQuery = new RangeQueryBuilder("text").timeZone("CET"); brokenQuery.query(brokenRangeQuery); - specifications.add(new RatedRequest("broken_query", brokenQuery, indices, types, createRelevant("1"))); + RatedRequest brokenRequest = new RatedRequest("broken_query", createRelevant("1"), brokenQuery); + brokenRequest.setIndices(indices); + brokenRequest.setTypes(types); + specifications.add(brokenRequest); RankEvalSpec task = new RankEvalSpec(specifications, new Precision()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 6a344994a5a..b2a0a42bb90 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.test.ESTestCase; import org.junit.AfterClass; @@ -41,9 +42,12 @@ import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static java.util.Collections.emptyList; @@ -70,24 +74,16 @@ public class RankEvalSpecTests extends ESTestCase { searchRequestParsers = null; } - public void testRoundtripping() throws IOException { - List indices = new ArrayList<>(); - int size = randomIntBetween(0, 20); + private static List randomList(Supplier randomSupplier) { + List result = new ArrayList<>(); + int size = randomIntBetween(1, 20); for (int i = 0; i < size; i++) { - indices.add(randomAsciiOfLengthBetween(0, 50)); - } - - List types = new ArrayList<>(); - size = randomIntBetween(0, 20); - for (int i = 0; i < size; i++) { - types.add(randomAsciiOfLengthBetween(0, 50)); - } - List specs = new ArrayList<>(); - size = randomIntBetween(1, 2); // TODO I guess requests with no query spec should be rejected... - for (int i = 0; i < size; i++) { - specs.add(RatedRequestsTests.createTestItem(indices, types)); + result.add(randomSupplier.get()); } + return result; + } + private RankEvalSpec createTestItem() throws IOException { RankedListQualityMetric metric; if (randomBoolean()) { metric = PrecisionTests.createTestItem(); @@ -95,8 +91,8 @@ public class RankEvalSpecTests extends ESTestCase { metric = DiscountedCumulativeGainTests.createTestItem(); } - RankEvalSpec testItem = new RankEvalSpec(specs, metric); - + Script template = null; + List ratedRequests = null; if (randomBoolean()) { final Map params = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("key", "value"); ScriptType scriptType = randomFrom(ScriptType.values()); @@ -112,9 +108,25 @@ public class RankEvalSpecTests extends ESTestCase { script = randomAsciiOfLengthBetween(1, 5); } - testItem.setTemplate(new Script(scriptType, randomFrom("_lang1", "_lang2"), script, params)); + template = new Script(scriptType, randomFrom("_lang1", "_lang2"), script, params); + + Map templateParams = new HashMap<>(); + templateParams.put("key", "value"); + RatedRequest ratedRequest = new RatedRequest( + "id", Arrays.asList(RatedDocumentTests.createRatedDocument()), templateParams); + ratedRequests = Arrays.asList(ratedRequest); + } else { + RatedRequest ratedRequest = new RatedRequest( + "id", Arrays.asList(RatedDocumentTests.createRatedDocument()), new SearchSourceBuilder()); + ratedRequests = Arrays.asList(ratedRequest); } + return new RankEvalSpec(ratedRequests, metric, template); + } + + public void testRoundtripping() throws IOException { + RankEvalSpec testItem = createTestItem(); + XContentBuilder shuffled = ESTestCase.shuffleXContent(testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); @@ -122,12 +134,34 @@ public class RankEvalSpecTests extends ESTestCase { RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, searchRequestParsers, null); - RankEvalSpec parsedItem = RankEvalSpec.parse(itemParser, rankContext, false); + RankEvalSpec parsedItem = RankEvalSpec.parse(itemParser, rankContext); // IRL these come from URL parameters - see RestRankEvalAction - parsedItem.getSpecifications().stream().forEach(e -> {e.setIndices(indices); e.setTypes(types);}); + // TODO Do we still need this? parsedItem.getRatedRequests().stream().forEach(e -> {e.setIndices(indices); e.setTypes(types);}); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); } + public void testMissingRatedRequestsFailsParsing() { + RankedListQualityMetric metric = new Precision(); + expectThrows(IllegalStateException.class, () -> new RankEvalSpec(new ArrayList<>(), metric)); + expectThrows(IllegalStateException.class, () -> new RankEvalSpec(null, metric)); + } + + public void testMissingMetricFailsParsing() { + List strings = Arrays.asList("value"); + List ratedRequests = randomList(() -> RatedRequestsTests.createTestItem(strings, strings)); + expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, null)); + } + + public void testMissingTemplateAndSearchRequestFailsParsing() { + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + Map params = new HashMap<>(); + params.put("key", "value"); + + RatedRequest request = new RatedRequest("id", ratedDocs, params); + List ratedRequests = Arrays.asList(request); + + expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, new Precision())); + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index e43b0e041d0..eddb8afc859 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -72,11 +72,8 @@ public class RatedRequestsTests extends ESTestCase { } public static RatedRequest createTestItem(List indices, List types) { - String specId = randomAsciiOfLength(50); + String requestId = randomAsciiOfLength(50); - SearchSourceBuilder testRequest = new SearchSourceBuilder(); - testRequest.size(randomInt()); - testRequest.query(new MatchAllQueryBuilder()); List ratedDocs = new ArrayList<>(); int size = randomIntBetween(0, 2); @@ -84,16 +81,17 @@ public class RatedRequestsTests extends ESTestCase { ratedDocs.add(RatedDocumentTests.createRatedDocument()); } - RatedRequest ratedRequest = new RatedRequest(specId, testRequest, indices, types, ratedDocs); - - + Map params = new HashMap<>(); + SearchSourceBuilder testRequest = null; if (randomBoolean()) { - Map params = new HashMap<>(); int randomSize = randomIntBetween(1, 10); for (int i = 0; i < randomSize; i++) { params.put(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); } - ratedRequest.setParams(params); + } else { + testRequest = new SearchSourceBuilder(); + testRequest.size(randomInt()); + testRequest.query(new MatchAllQueryBuilder()); } List summaryFields = new ArrayList<>(); @@ -101,7 +99,21 @@ public class RatedRequestsTests extends ESTestCase { for (int i = 0; i < numSummaryFields; i++) { summaryFields.add(randomAsciiOfLength(5)); } - ratedRequest.setSummaryFields(summaryFields); + + RatedRequest ratedRequest = null; + if (params.size() == 0) { + ratedRequest = new RatedRequest(requestId, ratedDocs, testRequest); + ratedRequest.setIndices(indices); + ratedRequest.setTypes(types); + ratedRequest.setSummaryFields(summaryFields); + } else { + ratedRequest = new RatedRequest(requestId, ratedDocs, params); + ratedRequest.setIndices(indices); + ratedRequest.setTypes(types); + ratedRequest.setSummaryFields(summaryFields); + } + + return ratedRequest; } @@ -181,53 +193,101 @@ public class RatedRequestsTests extends ESTestCase { } private RatedRequest mutateTestItem(RatedRequest original) { - String specId = original.getSpecId(); - int size = original.getTestRequest().size(); + String id = original.getId(); + SearchSourceBuilder testRequest = original.getTestRequest(); List ratedDocs = original.getRatedDocs(); List indices = original.getIndices(); List types = original.getTypes(); Map params = original.getParams(); List summaryFields = original.getSummaryFields(); - SearchSourceBuilder testRequest = new SearchSourceBuilder(); - testRequest.size(size); - testRequest.query(new MatchAllQueryBuilder()); + int mutate = randomIntBetween(0, 6); + switch (mutate) { + case 0: + id = randomValueOtherThan(id, () -> randomAsciiOfLength(10)); + break; + case 1: + int size = randomValueOtherThan(testRequest.size(), () -> randomInt()); + testRequest = new SearchSourceBuilder(); + testRequest.size(size); + testRequest.query(new MatchAllQueryBuilder()); + break; + case 2: + ratedDocs = Arrays.asList( + randomValueOtherThanMany(ratedDocs::contains, () -> RatedDocumentTests.createRatedDocument())); + break; + case 3: + indices = Arrays.asList(randomValueOtherThanMany(indices::contains, () -> randomAsciiOfLength(10))); + break; + case 4: + types = Arrays.asList(randomValueOtherThanMany(types::contains, () -> randomAsciiOfLength(10))); + break; + case 5: + params = new HashMap<>(); + params.putAll(params); + params.put("one_more_key", "one_more_value"); + break; + case 6: + summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains, () -> randomAsciiOfLength(10))); + break; + default: + throw new IllegalStateException("Requested to modify more than available parameters."); + } - RatedRequest ratedRequest = new RatedRequest(specId, testRequest, indices, types, ratedDocs); + RatedRequest ratedRequest = new RatedRequest(id, ratedDocs, testRequest, params); ratedRequest.setIndices(indices); ratedRequest.setTypes(types); - ratedRequest.setParams(params); ratedRequest.setSummaryFields(summaryFields); - List mutators = new ArrayList<>(); - mutators.add(() -> ratedRequest.setSpecId(randomValueOtherThan(specId, () -> randomAsciiOfLength(10)))); - mutators.add(() -> ratedRequest.getTestRequest().size(randomValueOtherThan(size, () -> randomInt()))); - mutators.add(() -> ratedRequest.setRatedDocs( - Arrays.asList(randomValueOtherThanMany(ratedDocs::contains, () -> RatedDocumentTests.createRatedDocument())))); - mutators.add(() -> ratedRequest.setIndices( - Arrays.asList(randomValueOtherThanMany(indices::contains, () -> randomAsciiOfLength(10))))); - - HashMap modified = new HashMap<>(); - modified.putAll(params); - modified.put("one_more_key", "one_more_value"); - mutators.add(() -> ratedRequest.setParams(modified)); - - mutators.add(() -> ratedRequest.setSummaryFields( - Arrays.asList(randomValueOtherThanMany(summaryFields::contains, () -> randomAsciiOfLength(10))))); - - - randomFrom(mutators).run(); return ratedRequest; } public void testDuplicateRatedDocThrowsException() { - RatedRequest request = createTestItem(Arrays.asList("index"), Arrays.asList("type")); List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1), new RatedDocument(new DocumentKey("index1", "type1", "id1"), 5)); - IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> request.setRatedDocs(ratedDocs)); + + // search request set, no summary fields + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder())); assertEquals( "Found duplicate rated document key [{ \"_index\" : \"index1\", \"_type\" : \"type1\", \"_id\" : \"id1\"}]", ex.getMessage()); + // templated path, no summary fields + Map params = new HashMap<>(); + params.put("key", "value"); + ex = expectThrows( + IllegalArgumentException.class, + () -> new RatedRequest("id", ratedDocs, params)); + assertEquals( + "Found duplicate rated document key [{ \"_index\" : \"index1\", \"_type\" : \"type1\", \"_id\" : \"id1\"}]", + ex.getMessage()); + } + + public void testNullSummaryFieldsTreatment() { + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder()); + expectThrows(IllegalArgumentException.class, () -> request.setSummaryFields(null)); + } + + public void testNullParamsTreatment() { + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null); + assertNotNull(request.getParams()); + } + + public void testSettingParamsAndRequestThrows() { + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + Map params = new HashMap<>(); + params.put("key", "value"); + expectThrows(IllegalArgumentException.class, + () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), params)); + } + + public void testSettingNeitherParamsNorRequestThrows() { + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null)); + expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, new HashMap<>())); } public void testParseFromXContent() throws IOException { @@ -256,7 +316,7 @@ public class RatedRequestsTests extends ESTestCase { RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, searchRequestParsers, null); RatedRequest specification = RatedRequest.fromXContent(parser, rankContext); - assertEquals("my_qa_query", specification.getSpecId()); + assertEquals("my_qa_query", specification.getId()); assertNotNull(specification.getTestRequest()); List ratedDocs = specification.getRatedDocs(); assertEquals(3, ratedDocs.size()); diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java index 02213642d39..536249c0110 100644 --- a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java @@ -80,26 +80,29 @@ public class SmokeMultipleTemplatesIT extends ESIntegTestCase { List types = Arrays.asList(new String[] { "testtype" }); List specifications = new ArrayList<>(); - RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", null, indices, types, createRelevant("2", "3", "4", "5")); Map ams_params = new HashMap<>(); ams_params.put("querystring", "amsterdam"); - amsterdamRequest.setParams(ams_params); + RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", createRelevant("2", "3", "4", "5"), ams_params); + amsterdamRequest.setIndices(indices); + amsterdamRequest.setTypes(types); + specifications.add(amsterdamRequest); - RatedRequest berlinRequest = new RatedRequest("berlin_query", null, indices, types, createRelevant("1")); Map berlin_params = new HashMap<>(); berlin_params.put("querystring", "berlin"); - berlinRequest.setParams(berlin_params); + RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("1"), berlin_params); + berlinRequest.setIndices(indices); + berlinRequest.setTypes(types); specifications.add(berlinRequest); Precision metric = new Precision(); - RankEvalSpec task = new RankEvalSpec(specifications, metric); - task.setTemplate( + + Script template = new Script( ScriptType.INLINE, "mustache", "{\"query\": {\"match\": {\"text\": \"{{querystring}}\"}}}", - new HashMap<>())); - + new HashMap<>()); + RankEvalSpec task = new RankEvalSpec(specifications, metric, template); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml index 8ab994878a1..00af4f9f41e 100644 --- a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml @@ -39,7 +39,7 @@ indices.refresh: {} - do: - rank_eval_template: + rank_eval: body: { "template": { "inline": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval_template.json deleted file mode 100644 index abb21c273e4..00000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval_template.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "rank_eval_template": { - "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-rank-eval.html", - "methods": ["POST"], - "url": { - "path": "/_rank_eval/template", - "paths": ["/_rank_eval/template", "/{index}/_rank_eval/template", "/{index}/{type}/_rank_eval/template"], - "parts": { - "index": { - "type": "list", - "description" : "A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices" - }, - "type": { - "type" : "list", - "description" : "A comma-separated list of document types to search; leave empty to perform the operation on all types" - } - }, - "params": {} - }, - "body": { - "description": "The search definition using the Query DSL and the prototype for the eval request.", - "required": true - } - } -} From eeff9fb100e9bb960dff6ec762b744e1abf45c1f Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Tue, 13 Dec 2016 13:59:11 +0100 Subject: [PATCH 086/297] Adjust docs to reflect removed template endpoint The dedicated template endpoint for rank_eval queries was removed, reflect this in the docs as well. --- docs/reference/search/rank-eval.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index 0220e48a5cd..d967e761181 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -78,7 +78,7 @@ GET /twitter/tweet/_rank_eval [source,js] -------------------------------- -GET /twitter/tweet/_rank_eval/template +GET /twitter/tweet/_rank_eval { "template": { "inline": { From bdc32be8b70bd82177056b1410e3e9930b7ace63 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Mon, 19 Dec 2016 12:49:15 +0100 Subject: [PATCH 087/297] Support specifying multiple templates (#22139) Problem: We introduced the ability to shorten the rank eval request by using a template in #20231. When playing with the API it turned out that there might be use cases where - e.g. due to various heuristics - folks might want to translate the original user query into more than just one type of Elasticsearch query. Solution: Give each template an id that can later be referenced in the actual requests. Closes #21257 --- docs/reference/search/rank-eval.asciidoc | 17 ++-- .../index/rankeval/RankEvalSpec.java | 96 ++++++++++++++----- .../index/rankeval/RatedRequest.java | 38 ++++++-- .../index/rankeval/RestRankEvalAction.java | 3 - .../rankeval/TransportRankEvalAction.java | 14 ++- .../index/rankeval/RankEvalSpecTests.java | 17 ++-- .../index/rankeval/RatedRequestsTests.java | 38 ++++++-- .../smoketest/SmokeMultipleTemplatesIT.java | 27 ++++-- .../test/rank-eval/30_template.yaml | 6 +- 9 files changed, 185 insertions(+), 71 deletions(-) diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index d967e761181..d06a0cb912d 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -80,12 +80,14 @@ GET /twitter/tweet/_rank_eval -------------------------------- GET /twitter/tweet/_rank_eval { - "template": { - "inline": { - "query": { - "match": { - "{{field}}": { - "query": "{{query_string}}"}}}}}, <1> + "templates": [{ + "id": "match_query", + "template": { + "inline": { + "query": { + "match": { + "{{field}}": { + "query": "{{query_string}}"}}}}}}], <1> "requests": [ { "id": "JFK query", @@ -105,7 +107,8 @@ GET /twitter/tweet/_rank_eval "params": { "query_string": "JFK", <2> "field": "opening_text" <2> - } + }, + "template_id": "match_query" }], "metric": { "precision": { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 4626b604f71..e1d67aeacc6 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; @@ -34,6 +33,9 @@ import org.elasticsearch.script.Script; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; /** @@ -49,10 +51,9 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { /** Definition of the quality metric, e.g. precision at N */ private RankedListQualityMetric metric; /** optional: Template to base test requests on */ - @Nullable - private Script template; + private Map templates = new HashMap<>(); - public RankEvalSpec(Collection ratedRequests, RankedListQualityMetric metric, Script template) { + public RankEvalSpec(Collection ratedRequests, RankedListQualityMetric metric, Collection templates) { if (ratedRequests == null || ratedRequests.size() < 1) { throw new IllegalStateException( "Cannot evaluate ranking if no search requests with rated results are provided. Seen: " + ratedRequests); @@ -61,7 +62,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { throw new IllegalStateException( "Cannot evaluate ranking if no evaluation metric is provided."); } - if (template == null) { + if (templates == null || templates.size() < 1) { for (RatedRequest request : ratedRequests) { if (request.getTestRequest() == null) { throw new IllegalStateException( @@ -72,7 +73,11 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } this.ratedRequests = ratedRequests; this.metric = metric; - this.template = template; + if (templates != null) { + for (ScriptWithId idScript : templates) { + this.templates.put(idScript.id, idScript.script); + } + } } public RankEvalSpec(Collection ratedRequests, RankedListQualityMetric metric) { @@ -80,29 +85,31 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } public RankEvalSpec(StreamInput in) throws IOException { - int specSize = in.readInt(); + int specSize = in.readVInt(); ratedRequests = new ArrayList<>(specSize); for (int i = 0; i < specSize; i++) { ratedRequests.add(new RatedRequest(in)); } metric = in.readNamedWriteable(RankedListQualityMetric.class); - if (in.readBoolean()) { - template = new Script(in); + int size = in.readVInt(); + for (int i = 0; i < size; i++) { + String key = in.readString(); + Script value = new Script(in); + this.templates.put(key, value); } } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeInt(ratedRequests.size()); + out.writeVInt(ratedRequests.size()); for (RatedRequest spec : ratedRequests) { spec.writeTo(out); } out.writeNamedWriteable(metric); - if (template != null) { - out.writeBoolean(true); - template.writeTo(out); - } else { - out.writeBoolean(false); + out.writeVInt(templates.size()); + for (Entry entry : templates.entrySet()) { + out.writeString(entry.getKey()); + entry.getValue().writeTo(out); } } @@ -117,17 +124,17 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } /** Returns the template to base test requests on. */ - public Script getTemplate() { - return this.template; + public Map getTemplates() { + return this.templates; } - private static final ParseField TEMPLATE_FIELD = new ParseField("template"); + private static final ParseField TEMPLATES_FIELD = new ParseField("templates"); private static final ParseField METRIC_FIELD = new ParseField("metric"); private static final ParseField REQUESTS_FIELD = new ParseField("requests"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rank_eval", - a -> new RankEvalSpec((Collection) a[0], (RankedListQualityMetric) a[1], (Script) a[2])); + a -> new RankEvalSpec((Collection) a[0], (RankedListQualityMetric) a[1], (Collection) a[2])); static { PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> { @@ -144,25 +151,62 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } } , METRIC_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { try { - return Script.parse(p, c.getParseFieldMatcher(), "mustache"); + return ScriptWithId.fromXContent(p, c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } - }, TEMPLATE_FIELD); + }, TEMPLATES_FIELD); } public static RankEvalSpec parse(XContentParser parser, RankEvalContext context) throws IOException { return PARSER.apply(parser, context); } + + public static class ScriptWithId { + private Script script; + private String id; + + private static final ParseField TEMPLATE_FIELD = new ParseField("template"); + private static final ParseField TEMPLATE_ID_FIELD = new ParseField("id"); + + public ScriptWithId(String id, Script script) { + this.id = id; + this.script = script; + } + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("script_with_id", a -> new ScriptWithId((String) a[0], (Script) a[1])); + + public static ScriptWithId fromXContent(XContentParser parser, RankEvalContext context) throws IOException { + return PARSER.apply(parser, context); + } + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), TEMPLATE_ID_FIELD); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { + try { + return Script.parse(p, c.getParseFieldMatcher(), "mustache"); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + } + }, TEMPLATE_FIELD); + } + } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (this.template != null) { - builder.field(TEMPLATE_FIELD.getPreferredName(), this.template); + builder.startArray(TEMPLATES_FIELD.getPreferredName()); + for (Entry entry : templates.entrySet()) { + builder.startObject(); + builder.field(ScriptWithId.TEMPLATE_ID_FIELD.getPreferredName(), entry.getKey()); + builder.field(ScriptWithId.TEMPLATE_FIELD.getPreferredName(), entry.getValue()); + builder.endObject(); } + builder.endArray(); + builder.startArray(REQUESTS_FIELD.getPreferredName()); for (RatedRequest spec : this.ratedRequests) { spec.toXContent(builder, params); @@ -185,11 +229,11 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { return Objects.equals(ratedRequests, other.ratedRequests) && Objects.equals(metric, other.metric) && - Objects.equals(template, other.template); + Objects.equals(templates, other.templates); } @Override public final int hashCode() { - return Objects.hash(ratedRequests, metric, template); + return Objects.hash(ratedRequests, metric, templates); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 6c61ade5c6a..acc963ebe23 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -60,16 +60,27 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { private SearchSourceBuilder testRequest; /** Map of parameters to use for filling a query template, can be used instead of providing testRequest. */ private Map params = new HashMap<>(); + @Nullable + private String templateId; - public RatedRequest(String id, List ratedDocs, SearchSourceBuilder testRequest, Map params) { + public RatedRequest( + String id, List ratedDocs, SearchSourceBuilder testRequest, Map params, String templateId) { if (params != null && (params.size() > 0 && testRequest != null)) { throw new IllegalArgumentException( "Ambiguous rated request: Set both, verbatim test request and test request template parameters."); } + if (templateId != null && testRequest != null) { + throw new IllegalArgumentException( + "Ambiguous rated request: Set both, verbatim test request and test request template parameters."); + } if ((params == null || params.size() < 1) && testRequest == null) { throw new IllegalArgumentException( "Need to set at least test request or test request template parameters."); } + if ((params != null && params.size() > 0) && templateId == null) { + throw new IllegalArgumentException( + "If template parameters are supplied need to set id of template to apply them to too."); + } // No documents with same _index/_type/id allowed. Set docKeys = new HashSet<>(); for (RatedDocument doc : ratedDocs) { @@ -86,14 +97,15 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { if (params != null) { this.params = params; } + this.templateId = templateId; } - public RatedRequest(String id, List ratedDocs, Map params) { - this(id, ratedDocs, null, params); + public RatedRequest(String id, List ratedDocs, Map params, String templateId) { + this(id, ratedDocs, null, params, templateId); } public RatedRequest(String id, List ratedDocs, SearchSourceBuilder testRequest) { - this(id, ratedDocs, testRequest, new HashMap<>()); + this(id, ratedDocs, testRequest, new HashMap<>(), null); } public RatedRequest(StreamInput in) throws IOException { @@ -121,6 +133,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { for (int i = 0; i < summaryFieldsSize; i++) { this.summaryFields.add(in.readString()); } + this.templateId = in.readOptionalString(); } @Override @@ -145,6 +158,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { for (String fieldName : summaryFields) { out.writeString(fieldName); } + out.writeOptionalString(this.templateId); } public SearchSourceBuilder getTestRequest() { @@ -181,6 +195,10 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { return this.params; } + public String getTemplateId() { + return this.templateId; + } + /** Returns a list of fields that are included in the docs summary of matched documents. */ public List getSummaryFields() { return summaryFields; @@ -198,10 +216,11 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { private static final ParseField RATINGS_FIELD = new ParseField("ratings"); private static final ParseField PARAMS_FIELD = new ParseField("params"); private static final ParseField FIELDS_FIELD = new ParseField("summary_fields"); + private static final ParseField TEMPLATE_ID_FIELD = new ParseField("template_id"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("requests", a -> new RatedRequest( - (String) a[0], (List) a[1], (SearchSourceBuilder) a[2], (Map) a[3])); + (String) a[0], (List) a[1], (SearchSourceBuilder) a[2], (Map) a[3], (String) a[4])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD); @@ -227,6 +246,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { } }, PARAMS_FIELD); PARSER.declareStringArray(RatedRequest::setSummaryFields, FIELDS_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TEMPLATE_ID_FIELD); } /** @@ -278,6 +298,9 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { } builder.endArray(); } + if (this.templateId != null) { + builder.field(TEMPLATE_ID_FIELD.getPreferredName(), this.templateId); + } builder.endObject(); return builder; } @@ -299,11 +322,12 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { Objects.equals(types, other.types) && Objects.equals(summaryFields, other.summaryFields) && Objects.equals(ratedDocs, other.ratedDocs) && - Objects.equals(params, other.params); + Objects.equals(params, other.params) && + Objects.equals(templateId, other.templateId); } @Override public final int hashCode() { - return Objects.hash(id, testRequest, indices, types, summaryFields, ratedDocs, params); + return Objects.hash(id, testRequest, indices, types, summaryFields, ratedDocs, params, templateId); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index fb042e1a8d6..275e15b1dda 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -21,16 +21,13 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchRequestParsers; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 9359e72c6e2..b95e66daaf7 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; @@ -46,6 +47,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -84,9 +86,11 @@ public class TransportRankEvalAction extends HandledTransportAction partialResults = new ConcurrentHashMap<>(specifications.size()); Map errors = new ConcurrentHashMap<>(specifications.size()); - CompiledScript scriptWithoutParams = null; - if (qualityTask.getTemplate() != null) { - scriptWithoutParams = scriptService.compile(qualityTask.getTemplate(), ScriptContext.Standard.SEARCH, new HashMap<>()); + Map scriptsWithoutParams = new HashMap<>(); + for (Entry entry : qualityTask.getTemplates().entrySet()) { + scriptsWithoutParams.put( + entry.getKey(), + scriptService.compile(entry.getValue(), ScriptContext.Standard.SEARCH, new HashMap<>())); } for (RatedRequest ratedRequest : specifications) { final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask.getMetric(), ratedRequest, @@ -94,7 +98,9 @@ public class TransportRankEvalAction extends HandledTransportAction params = ratedRequest.getParams(); - String resolvedRequest = ((BytesReference) (scriptService.executable(scriptWithoutParams, params).run())).utf8ToString(); + String templateId = ratedRequest.getTemplateId(); + CompiledScript compiled = scriptsWithoutParams.get(templateId); + String resolvedRequest = ((BytesReference) (scriptService.executable(compiled, params).run())).utf8ToString(); try (XContentParser subParser = XContentFactory.xContent(resolvedRequest).createParser(resolvedRequest)) { QueryParseContext parseContext = new QueryParseContext(searchRequestParsers.queryParsers, subParser, parseFieldMatcher); ratedSearchSource = SearchSourceBuilder.fromXContent(parseContext, searchRequestParsers.aggParsers, diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index b2a0a42bb90..5f46d22575c 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -43,8 +44,10 @@ import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -91,8 +94,9 @@ public class RankEvalSpecTests extends ESTestCase { metric = DiscountedCumulativeGainTests.createTestItem(); } - Script template = null; List ratedRequests = null; + Collection templates = null; + if (randomBoolean()) { final Map params = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("key", "value"); ScriptType scriptType = randomFrom(ScriptType.values()); @@ -108,20 +112,21 @@ public class RankEvalSpecTests extends ESTestCase { script = randomAsciiOfLengthBetween(1, 5); } - template = new Script(scriptType, randomFrom("_lang1", "_lang2"), script, params); + templates = new HashSet<>(); + templates.add( + new ScriptWithId("templateId", new Script(scriptType, randomFrom("_lang1", "_lang2"), script, params))); Map templateParams = new HashMap<>(); templateParams.put("key", "value"); RatedRequest ratedRequest = new RatedRequest( - "id", Arrays.asList(RatedDocumentTests.createRatedDocument()), templateParams); + "id", Arrays.asList(RatedDocumentTests.createRatedDocument()), templateParams, "templateId"); ratedRequests = Arrays.asList(ratedRequest); } else { RatedRequest ratedRequest = new RatedRequest( "id", Arrays.asList(RatedDocumentTests.createRatedDocument()), new SearchSourceBuilder()); ratedRequests = Arrays.asList(ratedRequest); } - - return new RankEvalSpec(ratedRequests, metric, template); + return new RankEvalSpec(ratedRequests, metric, templates); } public void testRoundtripping() throws IOException { @@ -159,7 +164,7 @@ public class RankEvalSpecTests extends ESTestCase { Map params = new HashMap<>(); params.put("key", "value"); - RatedRequest request = new RatedRequest("id", ratedDocs, params); + RatedRequest request = new RatedRequest("id", ratedDocs, params, "templateId"); List ratedRequests = Arrays.asList(request); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, new Precision())); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index eddb8afc859..67b7a65e338 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -107,7 +107,7 @@ public class RatedRequestsTests extends ESTestCase { ratedRequest.setTypes(types); ratedRequest.setSummaryFields(summaryFields); } else { - ratedRequest = new RatedRequest(requestId, ratedDocs, params); + ratedRequest = new RatedRequest(requestId, ratedDocs, params, randomAsciiOfLength(5)); ratedRequest.setIndices(indices); ratedRequest.setTypes(types); ratedRequest.setSummaryFields(summaryFields); @@ -200,8 +200,9 @@ public class RatedRequestsTests extends ESTestCase { List types = original.getTypes(); Map params = original.getParams(); List summaryFields = original.getSummaryFields(); + String templateId = original.getTemplateId(); - int mutate = randomIntBetween(0, 6); + int mutate = randomIntBetween(0, 7); switch (mutate) { case 0: id = randomValueOtherThan(id, () -> randomAsciiOfLength(10)); @@ -230,11 +231,14 @@ public class RatedRequestsTests extends ESTestCase { case 6: summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains, () -> randomAsciiOfLength(10))); break; + case 7: + templateId = randomValueOtherThan(templateId, () -> randomAsciiOfLength(5)); + break; default: throw new IllegalStateException("Requested to modify more than available parameters."); } - RatedRequest ratedRequest = new RatedRequest(id, ratedDocs, testRequest, params); + RatedRequest ratedRequest = new RatedRequest(id, ratedDocs, testRequest, params, templateId); ratedRequest.setIndices(indices); ratedRequest.setTypes(types); ratedRequest.setSummaryFields(summaryFields); @@ -258,7 +262,7 @@ public class RatedRequestsTests extends ESTestCase { params.put("key", "value"); ex = expectThrows( IllegalArgumentException.class, - () -> new RatedRequest("id", ratedDocs, params)); + () -> new RatedRequest("id", ratedDocs, params, "templateId")); assertEquals( "Found duplicate rated document key [{ \"_index\" : \"index1\", \"_type\" : \"type1\", \"_id\" : \"id1\"}]", ex.getMessage()); @@ -272,7 +276,7 @@ public class RatedRequestsTests extends ESTestCase { public void testNullParamsTreatment() { List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); - RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null); + RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, null); assertNotNull(request.getParams()); } @@ -281,13 +285,33 @@ public class RatedRequestsTests extends ESTestCase { Map params = new HashMap<>(); params.put("key", "value"); expectThrows(IllegalArgumentException.class, - () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), params)); + () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), params, null)); } public void testSettingNeitherParamsNorRequestThrows() { List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null)); - expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, new HashMap<>())); + expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, new HashMap<>(), "templateId")); + } + + public void testSettingParamsWithoutTemplateIdThrows() { + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + Map params = new HashMap<>(); + params.put("key", "value"); + expectThrows(IllegalArgumentException.class, + () -> new RatedRequest("id", ratedDocs, null, params, null)); + } + + public void testSettingTemplateIdAndRequestThrows() { + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + expectThrows(IllegalArgumentException.class, + () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, "templateId")); + } + + public void testSettingTemplateIdNoParamsThrows() { + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + expectThrows(IllegalArgumentException.class, + () -> new RatedRequest("id", ratedDocs, null, null, "templateId")); } public void testParseFromXContent() throws IOException { diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java index 536249c0110..09da73a1f7f 100644 --- a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.rankeval.RankEvalResponse; import org.elasticsearch.index.rankeval.RankEvalSpec; import org.elasticsearch.index.rankeval.RatedDocument; import org.elasticsearch.index.rankeval.RatedRequest; +import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -39,12 +40,16 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; public class SmokeMultipleTemplatesIT extends ESIntegTestCase { + private static final String MATCH_TEMPLATE = "match_template"; + @Override protected Collection> transportClientPlugins() { return Arrays.asList(RankEvalPlugin.class); @@ -82,7 +87,8 @@ public class SmokeMultipleTemplatesIT extends ESIntegTestCase { List specifications = new ArrayList<>(); Map ams_params = new HashMap<>(); ams_params.put("querystring", "amsterdam"); - RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", createRelevant("2", "3", "4", "5"), ams_params); + RatedRequest amsterdamRequest = new RatedRequest( + "amsterdam_query", createRelevant("2", "3", "4", "5"), ams_params, MATCH_TEMPLATE); amsterdamRequest.setIndices(indices); amsterdamRequest.setTypes(types); @@ -90,19 +96,24 @@ public class SmokeMultipleTemplatesIT extends ESIntegTestCase { Map berlin_params = new HashMap<>(); berlin_params.put("querystring", "berlin"); - RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("1"), berlin_params); + RatedRequest berlinRequest = new RatedRequest( + "berlin_query", createRelevant("1"), berlin_params, MATCH_TEMPLATE); berlinRequest.setIndices(indices); berlinRequest.setTypes(types); specifications.add(berlinRequest); Precision metric = new Precision(); - Script template = - new Script( - ScriptType.INLINE, - "mustache", "{\"query\": {\"match\": {\"text\": \"{{querystring}}\"}}}", - new HashMap<>()); - RankEvalSpec task = new RankEvalSpec(specifications, metric, template); + ScriptWithId template = + new ScriptWithId( + MATCH_TEMPLATE, + new Script( + ScriptType.INLINE, + "mustache", "{\"query\": {\"match\": {\"text\": \"{{querystring}}\"}}}", + new HashMap<>())); + Set templates = new HashSet<>(); + templates.add(template); + RankEvalSpec task = new RankEvalSpec(specifications, metric, templates); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml index 00af4f9f41e..40db3724a9a 100644 --- a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml @@ -41,13 +41,12 @@ - do: rank_eval: body: { - "template": { - "inline": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" - }, + "templates": [ { "id": "match", "template": {"inline": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" }} ], "requests" : [ { "id": "amsterdam_query", "params": { "query_string": "amsterdam" }, + "template_id": "match", "ratings": [ {"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 0}, {"_index": "foo", "_type": "bar", "_id": "doc2", "rating": 1}, @@ -56,6 +55,7 @@ { "id" : "berlin_query", "params": { "query_string": "berlin" }, + "template_id": "match", "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] } ], From 46c30e6bc3454cd740b76f4b0b8ceaeb4d8ef1a3 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Mon, 19 Dec 2016 13:05:49 +0100 Subject: [PATCH 088/297] Make maximum number of parallel search requests configurable. (#22192) Problem: So far all rank eval requests are being executed in parallel. If there are more than the search thread pool can handle, or if there are other search requests executed in parallel rank eval can fail. Solution: Make number of max_concurrent_searches configurable. Name of configuration parameter is analogous to msearch. Default max_concurrent_searches set to 10: Rank_eval isn't particularly time critical so trying to avoid being more clever than probably needed here. Can set this value through the API to a higher value anytime. Fixes #21403 --- docs/reference/search/rank-eval.asciidoc | 5 ++- .../index/rankeval/RankEvalSpec.java | 24 ++++++++++- .../rankeval/TransportRankEvalAction.java | 40 +++++++++++++++---- .../index/rankeval/RankEvalSpecTests.java | 4 +- 4 files changed, 62 insertions(+), 11 deletions(-) diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index d06a0cb912d..ee21ca19aa9 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -57,7 +57,8 @@ GET /twitter/tweet/_rank_eval }], "metric": { <9> "reciprocal_rank": {} - } + }, + "max_concurrent_searches": 10 <10> } ------------------------------ // CONSOLE @@ -72,6 +73,8 @@ GET /twitter/tweet/_rank_eval <7> The index where the rated document lives. <8> For a verbose response, specify which properties of a search hit should be returned in addition to index/type/id. <9> A metric to use for evaluation. See below for a list. +<10> Maximum number of search requests to execute in parallel. Set to 10 by +default. == Template based ranking evaluation diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index e1d67aeacc6..81919c87a01 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -50,7 +50,11 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { private Collection ratedRequests = new ArrayList<>(); /** Definition of the quality metric, e.g. precision at N */ private RankedListQualityMetric metric; - /** optional: Template to base test requests on */ + /** Maximum number of requests to execute in parallel. */ + private int maxConcurrentSearches = MAX_CONCURRENT_SEARCHES; + /** Default max number of requests. */ + private static final int MAX_CONCURRENT_SEARCHES = 10; + /** optional: Templates to base test requests on */ private Map templates = new HashMap<>(); public RankEvalSpec(Collection ratedRequests, RankedListQualityMetric metric, Collection templates) { @@ -97,6 +101,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { Script value = new Script(in); this.templates.put(key, value); } + maxConcurrentSearches = in.readVInt(); } @Override @@ -111,6 +116,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { out.writeString(entry.getKey()); entry.getValue().writeTo(out); } + out.writeVInt(maxConcurrentSearches); } /** Returns the metric to use for quality evaluation.*/ @@ -127,10 +133,21 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { public Map getTemplates() { return this.templates; } + + /** Returns the max concurrent searches allowed. */ + public int getMaxConcurrentSearches() { + return this.maxConcurrentSearches; + } + + /** Set the max concurrent searches allowed. */ + public void setMaxConcurrentSearches(int maxConcurrentSearches) { + this.maxConcurrentSearches = maxConcurrentSearches; + } private static final ParseField TEMPLATES_FIELD = new ParseField("templates"); private static final ParseField METRIC_FIELD = new ParseField("metric"); private static final ParseField REQUESTS_FIELD = new ParseField("requests"); + private static final ParseField MAX_CONCURRENT_SEARCHES_FIELD = new ParseField("max_concurrent_searches"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rank_eval", @@ -158,6 +175,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } }, TEMPLATES_FIELD); + PARSER.declareInt(RankEvalSpec::setMaxConcurrentSearches, MAX_CONCURRENT_SEARCHES_FIELD); } public static RankEvalSpec parse(XContentParser parser, RankEvalContext context) throws IOException { @@ -213,6 +231,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } builder.endArray(); builder.field(METRIC_FIELD.getPreferredName(), this.metric); + builder.field(MAX_CONCURRENT_SEARCHES_FIELD.getPreferredName(), maxConcurrentSearches); builder.endObject(); return builder; } @@ -229,11 +248,12 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { return Objects.equals(ratedRequests, other.ratedRequests) && Objects.equals(metric, other.metric) && + Objects.equals(maxConcurrentSearches, other.maxConcurrentSearches) && Objects.equals(templates, other.templates); } @Override public final int hashCode() { - return Objects.hash(ratedRequests, metric, templates); + return Objects.hash(ratedRequests, metric, templates, maxConcurrentSearches); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index b95e66daaf7..10e84e68d58 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -47,8 +47,10 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; /** @@ -65,6 +67,7 @@ public class TransportRankEvalAction extends HandledTransportAction taskQueue = new ConcurrentLinkedQueue<>(); @Inject public TransportRankEvalAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, @@ -81,10 +84,10 @@ public class TransportRankEvalAction extends HandledTransportAction listener) { RankEvalSpec qualityTask = request.getRankEvalSpec(); - Collection specifications = qualityTask.getRatedRequests(); - AtomicInteger responseCounter = new AtomicInteger(specifications.size()); - Map partialResults = new ConcurrentHashMap<>(specifications.size()); - Map errors = new ConcurrentHashMap<>(specifications.size()); + Collection ratedRequests = qualityTask.getRatedRequests(); + AtomicInteger responseCounter = new AtomicInteger(ratedRequests.size()); + Map partialResults = new ConcurrentHashMap<>(ratedRequests.size()); + Map errors = new ConcurrentHashMap<>(ratedRequests.size()); Map scriptsWithoutParams = new HashMap<>(); for (Entry entry : qualityTask.getTemplates().entrySet()) { @@ -92,7 +95,8 @@ public class TransportRankEvalAction extends HandledTransportAction())); } - for (RatedRequest ratedRequest : specifications) { + + for (RatedRequest ratedRequest : ratedRequests) { final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask.getMetric(), ratedRequest, partialResults, errors, responseCounter); SearchSourceBuilder ratedSearchSource = ratedRequest.getTestRequest(); @@ -124,11 +128,28 @@ public class TransportRankEvalAction extends HandledTransportAction { + private class RequestTask { + public SearchRequest request; + public RankEvalActionListener searchListener; + + public RequestTask(SearchRequest request, RankEvalActionListener listener) { + this.request = request; + this.searchListener = listener; + } + } + + public class RankEvalActionListener implements ActionListener { private ActionListener listener; private RatedRequest specification; @@ -165,6 +186,11 @@ public class TransportRankEvalAction extends HandledTransportAction Date: Mon, 19 Dec 2016 15:15:22 +0100 Subject: [PATCH 089/297] RankEvaluation: Add mutation based testing to RankEvalSpec (#22258) RankEvaluation: Add mutation based testing to RankEvalSpec, also fix RatedRequestsTests that were failing intermittently. --- .../index/rankeval/RankEvalSpecTests.java | 83 ++++++++++++++++++- .../index/rankeval/RatedRequestsTests.java | 29 ++++--- 2 files changed, 98 insertions(+), 14 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index acdbcd7ee60..3386caa38ee 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ParseFieldRegistry; import org.elasticsearch.common.xcontent.ToXContent; @@ -27,6 +28,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId; import org.elasticsearch.indices.query.IndicesQueriesRegistry; @@ -50,6 +53,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.function.Supplier; import static java.util.Collections.emptyList; @@ -131,7 +135,7 @@ public class RankEvalSpecTests extends ESTestCase { return spec; } - public void testRoundtripping() throws IOException { + public void testXContentRoundtrip() throws IOException { RankEvalSpec testItem = createTestItem(); XContentBuilder shuffled = ESTestCase.shuffleXContent(testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); @@ -149,6 +153,83 @@ public class RankEvalSpecTests extends ESTestCase { assertEquals(testItem.hashCode(), parsedItem.hashCode()); } + public void testSerialization() throws IOException { + RankEvalSpec original = createTestItem(); + + List namedWriteables = new ArrayList<>(); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry( + RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); + + + RankEvalSpec deserialized = RankEvalTestHelper.copy(original, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables)); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + + public void testEqualsAndHash() throws IOException { + RankEvalSpec testItem = createTestItem(); + + List namedWriteables = new ArrayList<>(); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry( + RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); + + RankEvalSpec mutant = RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables)); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(mutant), + RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables))); + } + + private RankEvalSpec mutateTestItem(RankEvalSpec mutant) { + Collection ratedRequests = mutant.getRatedRequests(); + RankedListQualityMetric metric = mutant.getMetric(); + Map templates = mutant.getTemplates(); + + int mutate = randomIntBetween(0, 2); + switch (mutate) { + case 0: + RatedRequest request = RatedRequestsTests.createTestItem(new ArrayList<>(), new ArrayList<>()); + ratedRequests.add(request); + break; + case 1: + if (metric instanceof Precision) { + metric = new DiscountedCumulativeGain(); + } else { + metric = new Precision(); + } + break; + case 2: + if (templates.size() > 0) { + if (randomBoolean()) { + templates = null; + } else { + String mutatedTemplate = randomAsciiOfLength(10); + templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, new HashMap<>())); + + } + } else { + String mutatedTemplate = randomValueOtherThanMany(templates::containsValue, () -> randomAsciiOfLength(10)); + templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, new HashMap<>())); + } + break; + default: + throw new IllegalStateException("Requested to modify more than available parameters."); + } + + List scripts = new ArrayList<>(); + for (Entry entry : templates.entrySet()) { + scripts.add(new ScriptWithId(entry.getKey(), entry.getValue())); + } + + RankEvalSpec result = new RankEvalSpec(ratedRequests, metric, scripts); + return result; + } + public void testMissingRatedRequestsFailsParsing() { RankedListQualityMetric metric = new Precision(); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(new ArrayList<>(), metric)); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 67b7a65e338..4844c21c299 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -202,16 +202,27 @@ public class RatedRequestsTests extends ESTestCase { List summaryFields = original.getSummaryFields(); String templateId = original.getTemplateId(); - int mutate = randomIntBetween(0, 7); + int mutate = randomIntBetween(0, 5); switch (mutate) { case 0: id = randomValueOtherThan(id, () -> randomAsciiOfLength(10)); break; case 1: - int size = randomValueOtherThan(testRequest.size(), () -> randomInt()); - testRequest = new SearchSourceBuilder(); - testRequest.size(size); - testRequest.query(new MatchAllQueryBuilder()); + if (testRequest != null) { + int size = randomValueOtherThan(testRequest.size(), () -> randomInt()); + testRequest = new SearchSourceBuilder(); + testRequest.size(size); + testRequest.query(new MatchAllQueryBuilder()); + } else { + if (randomBoolean()) { + Map mutated = new HashMap<>(); + mutated.putAll(params); + mutated.put("one_more_key", "one_more_value"); + params = mutated; + } else { + templateId = randomValueOtherThan(templateId, () -> randomAsciiOfLength(5)); + } + } break; case 2: ratedDocs = Arrays.asList( @@ -224,16 +235,8 @@ public class RatedRequestsTests extends ESTestCase { types = Arrays.asList(randomValueOtherThanMany(types::contains, () -> randomAsciiOfLength(10))); break; case 5: - params = new HashMap<>(); - params.putAll(params); - params.put("one_more_key", "one_more_value"); - break; - case 6: summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains, () -> randomAsciiOfLength(10))); break; - case 7: - templateId = randomValueOtherThan(templateId, () -> randomAsciiOfLength(5)); - break; default: throw new IllegalStateException("Requested to modify more than available parameters."); } From dde2a09ba5a20d493533d4b96d55418119e470b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 3 Feb 2017 20:53:30 +0100 Subject: [PATCH 090/297] Updating rank-eval module after major changes on master --- .../rankeval/DiscountedCumulativeGain.java | 7 +- .../index/rankeval/Precision.java | 7 +- .../index/rankeval/RankEvalContext.java | 80 ----------- .../index/rankeval/RankEvalPlugin.java | 15 +- .../index/rankeval/RankEvalResponse.java | 8 +- .../index/rankeval/RankEvalSpec.java | 34 ++--- .../rankeval/RankedListQualityMetric.java | 9 +- .../index/rankeval/RatedDocument.java | 7 +- .../index/rankeval/RatedRequest.java | 21 ++- .../index/rankeval/ReciprocalRank.java | 7 +- .../index/rankeval/RestRankEvalAction.java | 26 +--- .../rankeval/TransportRankEvalAction.java | 35 ++--- .../DiscountedCumulativeGainTests.java | 33 +++-- .../index/rankeval/PrecisionTests.java | 31 +++-- .../index/rankeval/RankEvalResponseTests.java | 2 - .../index/rankeval/RankEvalSpecTests.java | 86 ++++-------- .../index/rankeval/RankEvalTestHelper.java | 18 --- .../index/rankeval/RankEvalYamlIT.java | 3 +- .../index/rankeval/RatedDocumentTests.java | 20 ++- .../index/rankeval/RatedRequestsTests.java | 130 +++++++++--------- .../index/rankeval/ReciprocalRankTests.java | 24 ++-- .../build.gradle | 1 + ...stRankEvalWithMustacheYAMLTestSuiteIT.java | 3 +- 23 files changed, 236 insertions(+), 371 deletions(-) delete mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java index 65af4ec193a..27b4fc5fda1 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; @@ -141,7 +140,7 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { private static final ParseField NORMALIZE_FIELD = new ParseField("normalize"); private static final ParseField UNKNOWN_DOC_RATING_FIELD = new ParseField("unknown_doc_rating"); - private static final ObjectParser PARSER = + private static final ObjectParser PARSER = new ObjectParser<>("dcg_at", () -> new DiscountedCumulativeGain()); static { @@ -149,8 +148,8 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { PARSER.declareInt(DiscountedCumulativeGain::setUnknownDocRating, UNKNOWN_DOC_RATING_FIELD); } - public static DiscountedCumulativeGain fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { - return PARSER.apply(parser, matcher); + public static DiscountedCumulativeGain fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java index 1536388391b..40923fe1a0a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; @@ -49,7 +48,7 @@ public class Precision implements RankedListQualityMetric { private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); private static final ParseField IGNORE_UNLABELED_FIELD = new ParseField("ignore_unlabeled"); - private static final ObjectParser PARSER = new ObjectParser<>(NAME, Precision::new); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Precision::new); /** * This setting controls how unlabeled documents in the search hits are @@ -118,8 +117,8 @@ public class Precision implements RankedListQualityMetric { return ignoreUnlabeled; } - public static Precision fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { - return PARSER.apply(parser, matcher); + public static Precision fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); } /** Compute precisionAtN based on provided relevant document IDs. diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java deleted file mode 100644 index 0e9a617bb59..00000000000 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.rankeval; - -import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.ParseFieldMatcherSupplier; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.search.SearchExtRegistry; -import org.elasticsearch.search.SearchRequestParsers; -import org.elasticsearch.search.aggregations.AggregatorParsers; -import org.elasticsearch.search.suggest.Suggesters; - -public class RankEvalContext implements ParseFieldMatcherSupplier { - - private final SearchRequestParsers searchRequestParsers; - private final ParseFieldMatcher parseFieldMatcher; - private final QueryParseContext parseContext; - private final ScriptService scriptService; - - public RankEvalContext(ParseFieldMatcher parseFieldMatcher, QueryParseContext parseContext, SearchRequestParsers searchRequestParsers, - ScriptService scriptService) { - this.parseFieldMatcher = parseFieldMatcher; - this.searchRequestParsers = searchRequestParsers; - this.parseContext = parseContext; - this.scriptService = scriptService; - } - - public Suggesters getSuggesters() { - return searchRequestParsers.suggesters; - } - - public AggregatorParsers getAggs() { - return searchRequestParsers.aggParsers; - } - - public SearchRequestParsers getSearchRequestParsers() { - return searchRequestParsers; - } - - public ScriptService getScriptService() { - return scriptService; - } - - public SearchExtRegistry getSearchExtParsers() { - return searchRequestParsers.searchExtParsers; - } - - @Override - public ParseFieldMatcher getParseFieldMatcher() { - return this.parseFieldMatcher; - } - - public XContentParser parser() { - return this.parseContext.parser(); - } - - public QueryParseContext getParseContext() { - return this.parseContext; - } - -} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index 32e1dbb8486..ca2654fb943 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -21,15 +21,23 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Supplier; public class RankEvalPlugin extends Plugin implements ActionPlugin { @@ -38,9 +46,12 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin { return Arrays.asList(new ActionHandler<>(RankEvalAction.INSTANCE, TransportRankEvalAction.class)); } + @Override - public List> getRestHandlers() { - return Arrays.asList(RestRankEvalAction.class); + public List getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster) { + return Arrays.asList(new RestRankEvalAction(settings, restController)); } /** diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 23c4e27fb8c..5364fb87cdc 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -23,7 +23,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; @@ -42,7 +42,7 @@ import java.util.Map; * **/ //TODO instead of just returning averages over complete results, think of other statistics, micro avg, macro avg, partial results -public class RankEvalResponse extends ActionResponse implements ToXContent { +public class RankEvalResponse extends ActionResponse implements ToXContentObject { /**Average precision observed when issuing query intents with this specification.*/ private double qualityLevel; /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ @@ -113,6 +113,7 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.startObject("rank_eval"); builder.field("quality_level", qualityLevel); builder.startObject("details"); @@ -123,11 +124,12 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { builder.startObject("failures"); for (String key : failures.keySet()) { builder.startObject(key); - ElasticsearchException.renderException(builder, params, failures.get(key)); + ElasticsearchException.generateFailureXContent(builder, params, failures.get(key), true); builder.endObject(); } builder.endObject(); builder.endObject(); + builder.endObject(); return builder; } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 81919c87a01..e45ba14eda6 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -133,7 +133,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { public Map getTemplates() { return this.templates; } - + /** Returns the max concurrent searches allowed. */ public int getMaxConcurrentSearches() { return this.maxConcurrentSearches; @@ -149,43 +149,35 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { private static final ParseField REQUESTS_FIELD = new ParseField("requests"); private static final ParseField MAX_CONCURRENT_SEARCHES_FIELD = new ParseField("max_concurrent_searches"); @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rank_eval", a -> new RankEvalSpec((Collection) a[0], (RankedListQualityMetric) a[1], (Collection) a[2])); static { PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> { - try { - return RatedRequest.fromXContent(p, c); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); - } + return RatedRequest.fromXContent(p); } , REQUESTS_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { try { - return RankedListQualityMetric.fromXContent(p, c); + return RankedListQualityMetric.fromXContent(p); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } } , METRIC_FIELD); PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { - try { - return ScriptWithId.fromXContent(p, c); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); - } + return ScriptWithId.fromXContent(p); }, TEMPLATES_FIELD); PARSER.declareInt(RankEvalSpec::setMaxConcurrentSearches, MAX_CONCURRENT_SEARCHES_FIELD); } - public static RankEvalSpec parse(XContentParser parser, RankEvalContext context) throws IOException { - return PARSER.apply(parser, context); + public static RankEvalSpec parse(XContentParser parser) { + return PARSER.apply(parser, null); } - + public static class ScriptWithId { private Script script; private String id; - + private static final ParseField TEMPLATE_FIELD = new ParseField("template"); private static final ParseField TEMPLATE_ID_FIELD = new ParseField("id"); @@ -194,18 +186,18 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { this.script = script; } - private static final ConstructingObjectParser PARSER = + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("script_with_id", a -> new ScriptWithId((String) a[0], (Script) a[1])); - public static ScriptWithId fromXContent(XContentParser parser, RankEvalContext context) throws IOException { - return PARSER.apply(parser, context); + public static ScriptWithId fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); } static { PARSER.declareString(ConstructingObjectParser.constructorArg(), TEMPLATE_ID_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { try { - return Script.parse(p, c.getParseFieldMatcher(), "mustache"); + return Script.parse(p, "mustache"); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index cd1f6cb0e85..ea42f32ff07 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.xcontent.ToXContent; @@ -54,7 +53,7 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { * */ EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs); - static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { + static RankedListQualityMetric fromXContent(XContentParser parser) throws IOException { RankedListQualityMetric rc; Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { @@ -65,13 +64,13 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { // TODO maybe switch to using a plugable registry later? switch (metricName) { case Precision.NAME: - rc = Precision.fromXContent(parser, context); + rc = Precision.fromXContent(parser); break; case ReciprocalRank.NAME: - rc = ReciprocalRank.fromXContent(parser, context); + rc = ReciprocalRank.fromXContent(parser); break; case DiscountedCumulativeGain.NAME: - rc = DiscountedCumulativeGain.fromXContent(parser, context); + rc = DiscountedCumulativeGain.fromXContent(parser); break; default: throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index c6639619cac..0102de14436 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -21,7 +21,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -42,7 +41,7 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { public static final ParseField TYPE_FIELD = new ParseField("_type"); public static final ParseField INDEX_FIELD = new ParseField("_index"); - private static final ConstructingObjectParser PARSER = + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rated_document", a -> new RatedDocument((String) a[0], (String) a[1], (String) a[2], (Integer) a[3])); @@ -96,8 +95,8 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { out.writeVInt(rating); } - public static RatedDocument fromXContent(XContentParser parser, ParseFieldMatcherSupplier supplier) throws IOException { - return PARSER.apply(parser, supplier); + public static RatedDocument fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index acc963ebe23..395b77ef801 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; @@ -99,7 +100,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { } this.templateId = templateId; } - + public RatedRequest(String id, List ratedDocs, Map params, String templateId) { this(id, ratedDocs, null, params, templateId); } @@ -203,7 +204,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { public List getSummaryFields() { return summaryFields; } - + public void setSummaryFields(List summaryFields) { if (summaryFields == null) { throw new IllegalArgumentException("Setting summaryFields to null not allowed."); @@ -218,22 +219,18 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { private static final ParseField FIELDS_FIELD = new ParseField("summary_fields"); private static final ParseField TEMPLATE_ID_FIELD = new ParseField("template_id"); - private static final ConstructingObjectParser PARSER = + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("requests", a -> new RatedRequest( (String) a[0], (List) a[1], (SearchSourceBuilder) a[2], (Map) a[3], (String) a[4])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD); PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> { - try { - return RatedDocument.fromXContent(p, c); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex); - } + return RatedDocument.fromXContent(p); }, RATINGS_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { try { - return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters(), c.getSearchExtParsers()); + return SearchSourceBuilder.fromXContent(c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing request", ex); } @@ -270,8 +267,8 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { * "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ] * } */ - public static RatedRequest fromXContent(XContentParser parser, RankEvalContext context) throws IOException { - return PARSER.apply(parser, context); + public static RatedRequest fromXContent(XContentParser parser) { + return PARSER.apply(parser, new QueryParseContext(parser)); } @Override @@ -315,7 +312,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { } RatedRequest other = (RatedRequest) obj; - + return Objects.equals(id, other.id) && Objects.equals(testRequest, other.testRequest) && Objects.equals(indices, other.indices) && diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index a0238717169..2c4021ce2fc 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; @@ -117,15 +116,15 @@ public class ReciprocalRank implements RankedListQualityMetric { } private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); - private static final ObjectParser PARSER = new ObjectParser<>( + private static final ObjectParser PARSER = new ObjectParser<>( "reciprocal_rank", () -> new ReciprocalRank()); static { PARSER.declareInt(ReciprocalRank::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); } - public static ReciprocalRank fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { - return PARSER.apply(parser, matcher); + public static ReciprocalRank fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 275e15b1dda..3acba41e54f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -21,16 +21,12 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.search.SearchRequestParsers; import java.io.IOException; import java.util.Arrays; @@ -154,18 +150,10 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; * */ public class RestRankEvalAction extends BaseRestHandler { - private SearchRequestParsers searchRequestParsers; - private ScriptService scriptService; + //private ScriptService scriptService; - @Inject - public RestRankEvalAction( - Settings settings, - RestController controller, - SearchRequestParsers searchRequestParsers, - ScriptService scriptService) { + public RestRankEvalAction(Settings settings, RestController controller) { super(settings); - this.searchRequestParsers = searchRequestParsers; - this.scriptService = scriptService; controller.registerHandler(GET, "/_rank_eval", this); controller.registerHandler(POST, "/_rank_eval", this); controller.registerHandler(GET, "/{index}/_rank_eval", this); @@ -178,21 +166,17 @@ public class RestRankEvalAction extends BaseRestHandler { protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { RankEvalRequest rankEvalRequest = new RankEvalRequest(); try (XContentParser parser = request.contentOrSourceParamParser()) { - QueryParseContext parseContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher); - // TODO can we get rid of aggregators parsers and suggesters? - parseRankEvalRequest(rankEvalRequest, request, - new RankEvalContext(parseFieldMatcher, parseContext, searchRequestParsers, scriptService)); + parseRankEvalRequest(rankEvalRequest, request, parser); } return channel -> client.executeLocally(RankEvalAction.INSTANCE, rankEvalRequest, new RestToXContentListener(channel)); } - public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context) - throws IOException { + private static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, XContentParser parser) { List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); List types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type"))); RankEvalSpec spec = null; - spec = RankEvalSpec.parse(context.parser(), context); + spec = RankEvalSpec.parse(parser); for (RatedRequest specification : spec.getRatedRequests()) { specification.setIndices(indices); specification.setTypes(types); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 10e84e68d58..fafda7bb704 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -29,15 +29,15 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -47,12 +47,14 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Queue; import java.util.Map.Entry; +import java.util.Queue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; +import static org.elasticsearch.common.xcontent.XContentHelper.createParser; + /** * Instances of this class execute a collection of search intents (read: user supplied query parameters) against a set of * possible search requests (read: search specifications, expressed as query/search request templates) and compares the result @@ -66,17 +68,17 @@ import java.util.concurrent.atomic.AtomicInteger; public class TransportRankEvalAction extends HandledTransportAction { private Client client; private ScriptService scriptService; - private SearchRequestParsers searchRequestParsers; Queue taskQueue = new ConcurrentLinkedQueue<>(); - + private NamedXContentRegistry namedXContentRegistry; + @Inject public TransportRankEvalAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Client client, TransportService transportService, - SearchRequestParsers searchRequestParsers, ScriptService scriptService) { + ScriptService scriptService, NamedXContentRegistry namedXContentRegistry) { super(settings, RankEvalAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, RankEvalRequest::new); - this.searchRequestParsers = searchRequestParsers; this.scriptService = scriptService; + this.namedXContentRegistry = namedXContentRegistry; this.client = client; } @@ -92,8 +94,8 @@ public class TransportRankEvalAction extends HandledTransportAction scriptsWithoutParams = new HashMap<>(); for (Entry entry : qualityTask.getTemplates().entrySet()) { scriptsWithoutParams.put( - entry.getKey(), - scriptService.compile(entry.getValue(), ScriptContext.Standard.SEARCH, new HashMap<>())); + entry.getKey(), + scriptService.compile(entry.getValue(), ScriptContext.Standard.SEARCH)); } for (RatedRequest ratedRequest : ratedRequests) { @@ -104,11 +106,10 @@ public class TransportRankEvalAction extends HandledTransportAction params = ratedRequest.getParams(); String templateId = ratedRequest.getTemplateId(); CompiledScript compiled = scriptsWithoutParams.get(templateId); - String resolvedRequest = ((BytesReference) (scriptService.executable(compiled, params).run())).utf8ToString(); - try (XContentParser subParser = XContentFactory.xContent(resolvedRequest).createParser(resolvedRequest)) { - QueryParseContext parseContext = new QueryParseContext(searchRequestParsers.queryParsers, subParser, parseFieldMatcher); - ratedSearchSource = SearchSourceBuilder.fromXContent(parseContext, searchRequestParsers.aggParsers, - searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); + BytesReference resolvedRequest = (BytesReference) (scriptService.executable(compiled, params).run()); + try (XContentParser subParser = createParser(namedXContentRegistry, resolvedRequest, XContentType.JSON)) { + QueryParseContext parseContext = new QueryParseContext(subParser); + ratedSearchSource = SearchSourceBuilder.fromXContent(parseContext); } catch (IOException e) { listener.onFailure(e); } @@ -142,13 +143,13 @@ public class TransportRankEvalAction extends HandledTransportAction { private ActionListener listener; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 80f954f3914..209b8ca8fe3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -19,10 +19,13 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; @@ -192,10 +195,11 @@ public class DiscountedCumulativeGainTests extends ESTestCase { + " \"unknown_doc_rating\": 2,\n" + " \"normalize\": true\n" + "}"; - XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent); - DiscountedCumulativeGain dcgAt = DiscountedCumulativeGain.fromXContent(parser, () -> ParseFieldMatcher.STRICT); - assertEquals(2, dcgAt.getUnknownDocRating().intValue()); - assertEquals(true, dcgAt.getNormalize()); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) { + DiscountedCumulativeGain dcgAt = DiscountedCumulativeGain.fromXContent(parser); + assertEquals(2, dcgAt.getUnknownDocRating().intValue()); + assertEquals(true, dcgAt.getNormalize()); + } } public static DiscountedCumulativeGain createTestItem() { @@ -206,15 +210,18 @@ public class DiscountedCumulativeGainTests extends ESTestCase { } public void testXContentRoundtrip() throws IOException { DiscountedCumulativeGain testItem = createTestItem(); - XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); - itemParser.nextToken(); - itemParser.nextToken(); - DiscountedCumulativeGain parsedItem = DiscountedCumulativeGain.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); - assertNotSame(testItem, parsedItem); - assertEquals(testItem, parsedItem); - assertEquals(testItem.hashCode(), parsedItem.hashCode()); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + try (XContentParser itemParser = createParser(shuffled)) { + itemParser.nextToken(); + itemParser.nextToken(); + DiscountedCumulativeGain parsedItem = DiscountedCumulativeGain.fromXContent(itemParser); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + } } - + public void testSerialization() throws IOException { DiscountedCumulativeGain original = createTestItem(); DiscountedCumulativeGain deserialized = RankEvalTestHelper.copy(original, DiscountedCumulativeGain::new); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java index eec41ea2fec..77e6cfaff34 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java @@ -19,10 +19,13 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.Index; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; @@ -153,9 +156,10 @@ public class PrecisionTests extends ESTestCase { String xContent = " {\n" + " \"relevant_rating_threshold\" : 2" + "}"; - XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent); - Precision precicionAt = Precision.fromXContent(parser, () -> ParseFieldMatcher.STRICT); - assertEquals(2, precicionAt.getRelevantRatingThreshold()); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) { + Precision precicionAt = Precision.fromXContent(parser); + assertEquals(2, precicionAt.getRelevantRatingThreshold()); + } } public void testCombine() { @@ -183,15 +187,18 @@ public class PrecisionTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { Precision testItem = createTestItem(); - XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); - itemParser.nextToken(); - itemParser.nextToken(); - Precision parsedItem = Precision.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); - assertNotSame(testItem, parsedItem); - assertEquals(testItem, parsedItem); - assertEquals(testItem.hashCode(), parsedItem.hashCode()); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + try (XContentParser itemParser = createParser(shuffled)) { + itemParser.nextToken(); + itemParser.nextToken(); + Precision parsedItem = Precision.fromXContent(itemParser); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + } } - + public void testSerialization() throws IOException { Precision original = createTestItem(); Precision deserialized = RankEvalTestHelper.copy(original, Precision::new); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index fb24043dca1..fa1a512c54b 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -78,9 +78,7 @@ public class RankEvalResponseTests extends ESTestCase { if (ESTestCase.randomBoolean()) { builder.prettyPrint(); } - builder.startObject(); randomResponse.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); // TODO check the correctness of the output } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 3386caa38ee..b1ab9d9134a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -19,30 +19,19 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ParseFieldRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId; -import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.SearchRequestParsers; -import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.test.ESTestCase; -import org.junit.AfterClass; -import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; @@ -56,30 +45,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.function.Supplier; -import static java.util.Collections.emptyList; - public class RankEvalSpecTests extends ESTestCase { - private static SearchModule searchModule; - private static SearchRequestParsers searchRequestParsers; - - /** - * setup for the whole base test class - */ - @BeforeClass - public static void init() { - AggregatorParsers aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"), - new ParseFieldRegistry<>("aggregation_pipes")); - searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); - IndicesQueriesRegistry queriesRegistry = searchModule.getQueryParserRegistry(); - Suggesters suggesters = searchModule.getSuggesters(); - searchRequestParsers = new SearchRequestParsers(queriesRegistry, aggsParsers, suggesters, null); - } - - @AfterClass - public static void afterClass() throws Exception { - searchModule = null; - searchRequestParsers = null; - } private static List randomList(Supplier randomSupplier) { List result = new ArrayList<>(); @@ -90,7 +56,7 @@ public class RankEvalSpecTests extends ESTestCase { return result; } - private RankEvalSpec createTestItem() throws IOException { + private static RankEvalSpec createTestItem() throws IOException { RankedListQualityMetric metric; if (randomBoolean()) { metric = PrecisionTests.createTestItem(); @@ -118,7 +84,7 @@ public class RankEvalSpecTests extends ESTestCase { templates = new HashSet<>(); templates.add( - new ScriptWithId("templateId", new Script(scriptType, randomFrom("_lang1", "_lang2"), script, params))); + new ScriptWithId("templateId", new Script(scriptType, Script.DEFAULT_TEMPLATE_LANG, script, params))); Map templateParams = new HashMap<>(); templateParams.put("key", "value"); @@ -130,7 +96,7 @@ public class RankEvalSpecTests extends ESTestCase { "id", Arrays.asList(RatedDocumentTests.createRatedDocument()), new SearchSourceBuilder()); ratedRequests = Arrays.asList(ratedRequest); } - RankEvalSpec spec = new RankEvalSpec(ratedRequests, metric, templates); + RankEvalSpec spec = new RankEvalSpec(ratedRequests, metric, templates); maybeSet(spec::setMaxConcurrentSearches, randomInt(100)); return spec; } @@ -138,19 +104,18 @@ public class RankEvalSpecTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { RankEvalSpec testItem = createTestItem(); - XContentBuilder shuffled = ESTestCase.shuffleXContent(testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); - XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, shuffled.bytes())) { - QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); - RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, - searchRequestParsers, null); - - RankEvalSpec parsedItem = RankEvalSpec.parse(itemParser, rankContext); - // IRL these come from URL parameters - see RestRankEvalAction - // TODO Do we still need this? parsedItem.getRatedRequests().stream().forEach(e -> {e.setIndices(indices); e.setTypes(types);}); - assertNotSame(testItem, parsedItem); - assertEquals(testItem, parsedItem); - assertEquals(testItem.hashCode(), parsedItem.hashCode()); + RankEvalSpec parsedItem = RankEvalSpec.parse(parser); + // IRL these come from URL parameters - see RestRankEvalAction + // TODO Do we still need this? + // parsedItem.getRatedRequests().stream().forEach(e -> + // {e.setIndices(indices); e.setTypes(types);}); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + } } public void testSerialization() throws IOException { @@ -185,7 +150,7 @@ public class RankEvalSpecTests extends ESTestCase { RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables))); } - private RankEvalSpec mutateTestItem(RankEvalSpec mutant) { + private static RankEvalSpec mutateTestItem(RankEvalSpec mutant) { Collection ratedRequests = mutant.getRatedRequests(); RankedListQualityMetric metric = mutant.getMetric(); Map templates = mutant.getTemplates(); @@ -193,7 +158,7 @@ public class RankEvalSpecTests extends ESTestCase { int mutate = randomIntBetween(0, 2); switch (mutate) { case 0: - RatedRequest request = RatedRequestsTests.createTestItem(new ArrayList<>(), new ArrayList<>()); + RatedRequest request = RatedRequestsTests.createTestItem(new ArrayList<>(), new ArrayList<>(), true); ratedRequests.add(request); break; case 1: @@ -205,13 +170,8 @@ public class RankEvalSpecTests extends ESTestCase { break; case 2: if (templates.size() > 0) { - if (randomBoolean()) { - templates = null; - } else { - String mutatedTemplate = randomAsciiOfLength(10); - templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, new HashMap<>())); - - } + String mutatedTemplate = randomAsciiOfLength(10); + templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, new HashMap<>())); } else { String mutatedTemplate = randomValueOtherThanMany(templates::containsValue, () -> randomAsciiOfLength(10)); templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, new HashMap<>())); @@ -225,7 +185,7 @@ public class RankEvalSpecTests extends ESTestCase { for (Entry entry : templates.entrySet()) { scripts.add(new ScriptWithId(entry.getKey(), entry.getValue())); } - + RankEvalSpec result = new RankEvalSpec(ratedRequests, metric, scripts); return result; } @@ -235,10 +195,10 @@ public class RankEvalSpecTests extends ESTestCase { expectThrows(IllegalStateException.class, () -> new RankEvalSpec(new ArrayList<>(), metric)); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(null, metric)); } - + public void testMissingMetricFailsParsing() { List strings = Arrays.asList("value"); - List ratedRequests = randomList(() -> RatedRequestsTests.createTestItem(strings, strings)); + List ratedRequests = randomList(() -> RatedRequestsTests.createTestItem(strings, strings, randomBoolean())); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, null)); } @@ -249,7 +209,7 @@ public class RankEvalSpecTests extends ESTestCase { RatedRequest request = new RatedRequest("id", ratedDocs, params, "templateId"); List ratedRequests = Arrays.asList(request); - + expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, new Precision())); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java index 8be80707e06..be97ad8ff06 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java @@ -25,13 +25,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.Collections; @@ -45,17 +38,6 @@ import static org.junit.Assert.assertTrue; public class RankEvalTestHelper { - public static XContentParser roundtrip(ToXContent testItem) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(ESTestCase.randomFrom(XContentType.values())); - if (ESTestCase.randomBoolean()) { - builder.prettyPrint(); - } - testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); - XContentBuilder shuffled = ESTestCase.shuffleXContent(builder); - XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); - return itemParser; - } - public static void testHashCodeAndEquals(T testItem, T mutation, T secondCopy) { assertFalse("testItem is equal to null", testItem.equals(null)); assertFalse("testItem is equal to incompatible type", testItem.equals("")); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java index 09e1a906044..90c07bb244d 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java @@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -34,7 +33,7 @@ public class RankEvalYamlIT extends ESClientYamlSuiteTestCase { } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index f42c6e3e542..b100a21b38b 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -19,8 +19,11 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -38,13 +41,16 @@ public class RatedDocumentTests extends ESTestCase { public void testXContentParsing() throws IOException { RatedDocument testItem = createRatedDocument(); - XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); - RatedDocument parsedItem = RatedDocument.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); - assertNotSame(testItem, parsedItem); - assertEquals(testItem, parsedItem); - assertEquals(testItem.hashCode(), parsedItem.hashCode()); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + try (XContentParser itemParser = createParser(shuffled)) { + RatedDocument parsedItem = RatedDocument.fromXContent(itemParser); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + } } - + public void testSerialization() throws IOException { RatedDocument original = createRatedDocument(); RatedDocument deserialized = RankEvalTestHelper.copy(original, RatedDocument::new); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 4844c21c299..99035504785 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -19,21 +19,19 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ParseFieldRegistry; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.SearchRequestParsers; -import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.test.ESTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -44,36 +42,38 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.stream.Stream; import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; public class RatedRequestsTests extends ESTestCase { - private static SearchModule searchModule; - private static SearchRequestParsers searchRequestParsers; + private static NamedXContentRegistry xContentRegistry; /** * setup for the whole base test class */ @BeforeClass - public static void init() throws IOException { - AggregatorParsers aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"), - new ParseFieldRegistry<>("aggregation_pipes")); - searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); - IndicesQueriesRegistry queriesRegistry = searchModule.getQueryParserRegistry(); - Suggesters suggesters = searchModule.getSuggesters(); - searchRequestParsers = new SearchRequestParsers(queriesRegistry, aggsParsers, suggesters, null); + public static void init() { + xContentRegistry = new NamedXContentRegistry(Stream.of( + new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents().stream() + ).flatMap(Function.identity()).collect(toList())); } @AfterClass public static void afterClass() throws Exception { - searchModule = null; - searchRequestParsers = null; + xContentRegistry = null; } - public static RatedRequest createTestItem(List indices, List types) { - String requestId = randomAsciiOfLength(50); + @Override + protected NamedXContentRegistry xContentRegistry() { + return xContentRegistry; + } + public static RatedRequest createTestItem(List indices, List types, boolean forceRequest) { + String requestId = randomAsciiOfLength(50); List ratedDocs = new ArrayList<>(); int size = randomIntBetween(0, 2); @@ -83,15 +83,15 @@ public class RatedRequestsTests extends ESTestCase { Map params = new HashMap<>(); SearchSourceBuilder testRequest = null; - if (randomBoolean()) { + if (randomBoolean() || forceRequest) { + testRequest = new SearchSourceBuilder(); + testRequest.size(randomInt()); + testRequest.query(new MatchAllQueryBuilder()); + } else { int randomSize = randomIntBetween(1, 10); for (int i = 0; i < randomSize; i++) { params.put(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); } - } else { - testRequest = new SearchSourceBuilder(); - testRequest.size(randomInt()); - testRequest.query(new MatchAllQueryBuilder()); } List summaryFields = new ArrayList<>(); @@ -100,7 +100,7 @@ public class RatedRequestsTests extends ESTestCase { summaryFields.add(randomAsciiOfLength(5)); } - RatedRequest ratedRequest = null; + RatedRequest ratedRequest = null; if (params.size() == 0) { ratedRequest = new RatedRequest(requestId, ratedDocs, testRequest); ratedRequest.setIndices(indices); @@ -112,8 +112,6 @@ public class RatedRequestsTests extends ESTestCase { ratedRequest.setTypes(types); ratedRequest.setSummaryFields(summaryFields); } - - return ratedRequest; } @@ -130,22 +128,24 @@ public class RatedRequestsTests extends ESTestCase { types.add(randomAsciiOfLengthBetween(0, 50)); } - RatedRequest testItem = createTestItem(indices, types); - XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); - itemParser.nextToken(); + RatedRequest testItem = createTestItem(indices, types, randomBoolean()); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + try (XContentParser itemParser = createParser(shuffled)) { + itemParser.nextToken(); - QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); - RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, - searchRequestParsers, null); - - RatedRequest parsedItem = RatedRequest.fromXContent(itemParser, rankContext); - parsedItem.setIndices(indices); // IRL these come from URL parameters - see RestRankEvalAction - parsedItem.setTypes(types); // IRL these come from URL parameters - see RestRankEvalAction - assertNotSame(testItem, parsedItem); - assertEquals(testItem, parsedItem); - assertEquals(testItem.hashCode(), parsedItem.hashCode()); + RatedRequest parsedItem = RatedRequest.fromXContent(itemParser); + parsedItem.setIndices(indices); // IRL these come from URL + // parameters - see + // RestRankEvalAction + parsedItem.setTypes(types); // IRL these come from URL parameters - + // see RestRankEvalAction + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + } } - + public void testSerialization() throws IOException { List indices = new ArrayList<>(); int size = randomIntBetween(0, 20); @@ -159,7 +159,7 @@ public class RatedRequestsTests extends ESTestCase { types.add(randomAsciiOfLengthBetween(0, 50)); } - RatedRequest original = createTestItem(indices, types); + RatedRequest original = createTestItem(indices, types, randomBoolean()); List namedWriteables = new ArrayList<>(); namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); @@ -183,7 +183,7 @@ public class RatedRequestsTests extends ESTestCase { types.add(randomAsciiOfLengthBetween(0, 50)); } - RatedRequest testItem = createTestItem(indices, types); + RatedRequest testItem = createTestItem(indices, types, randomBoolean()); List namedWriteables = new ArrayList<>(); namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); @@ -270,19 +270,19 @@ public class RatedRequestsTests extends ESTestCase { "Found duplicate rated document key [{ \"_index\" : \"index1\", \"_type\" : \"type1\", \"_id\" : \"id1\"}]", ex.getMessage()); } - + public void testNullSummaryFieldsTreatment() { List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder()); expectThrows(IllegalArgumentException.class, () -> request.setSummaryFields(null)); } - + public void testNullParamsTreatment() { List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, null); assertNotNull(request.getParams()); } - + public void testSettingParamsAndRequestThrows() { List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); Map params = new HashMap<>(); @@ -290,7 +290,7 @@ public class RatedRequestsTests extends ESTestCase { expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), params, null)); } - + public void testSettingNeitherParamsNorRequestThrows() { List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null)); @@ -304,7 +304,7 @@ public class RatedRequestsTests extends ESTestCase { expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, params, null)); } - + public void testSettingTemplateIdAndRequestThrows() { List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); expectThrows(IllegalArgumentException.class, @@ -338,23 +338,21 @@ public class RatedRequestsTests extends ESTestCase { + " {\"_type\": \"testtype\", \"_index\": \"test\", \"_id\": \"2\", \"rating\" : 0 }, " + " {\"_id\": \"3\", \"_index\": \"test\", \"_type\": \"testtype\", \"rating\" : 1 }]\n" + "}"; - XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString); - QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, ParseFieldMatcher.STRICT); - RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, - searchRequestParsers, null); - RatedRequest specification = RatedRequest.fromXContent(parser, rankContext); - assertEquals("my_qa_query", specification.getId()); - assertNotNull(specification.getTestRequest()); - List ratedDocs = specification.getRatedDocs(); - assertEquals(3, ratedDocs.size()); - for (int i = 0; i < 3; i++) { - assertEquals("" + (i + 1), ratedDocs.get(i).getDocID()); - assertEquals("test", ratedDocs.get(i).getIndex()); - assertEquals("testtype", ratedDocs.get(i).getType()); - if (i == 1) { - assertEquals(0, ratedDocs.get(i).getRating()); - } else { - assertEquals(1, ratedDocs.get(i).getRating()); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, querySpecString)) { + RatedRequest specification = RatedRequest.fromXContent(parser); + assertEquals("my_qa_query", specification.getId()); + assertNotNull(specification.getTestRequest()); + List ratedDocs = specification.getRatedDocs(); + assertEquals(3, ratedDocs.size()); + for (int i = 0; i < 3; i++) { + assertEquals("" + (i + 1), ratedDocs.get(i).getDocID()); + assertEquals("test", ratedDocs.get(i).getIndex()); + assertEquals("testtype", ratedDocs.get(i).getType()); + if (i == 1) { + assertEquals(0, ratedDocs.get(i).getRating()); + } else { + assertEquals(1, ratedDocs.get(i).getRating()); + } } } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index b3c2e0d7cdc..c66f7ba45aa 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -19,9 +19,12 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionTests.Rating; import org.elasticsearch.search.SearchHit; @@ -124,13 +127,16 @@ public class ReciprocalRankTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { ReciprocalRank testItem = createTestItem(); - XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); - itemParser.nextToken(); - itemParser.nextToken(); - ReciprocalRank parsedItem = ReciprocalRank.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); - assertNotSame(testItem, parsedItem); - assertEquals(testItem, parsedItem); - assertEquals(testItem.hashCode(), parsedItem.hashCode()); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + try (XContentParser itemParser = createParser(shuffled)) { + itemParser.nextToken(); + itemParser.nextToken(); + ReciprocalRank parsedItem = ReciprocalRank.fromXContent(itemParser); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + } } /** @@ -145,7 +151,7 @@ public class ReciprocalRankTests extends ESTestCase { } return hits; } - + private ReciprocalRank createTestItem() { ReciprocalRank testItem = new ReciprocalRank(); testItem.setRelevantRatingThreshhold(randomIntBetween(0, 20)); diff --git a/qa/smoke-test-rank-eval-with-mustache/build.gradle b/qa/smoke-test-rank-eval-with-mustache/build.gradle index 4fdbaa04502..7274e65f4e1 100644 --- a/qa/smoke-test-rank-eval-with-mustache/build.gradle +++ b/qa/smoke-test-rank-eval-with-mustache/build.gradle @@ -17,6 +17,7 @@ * under the License. */ +apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java index 92c60b6f090..401bddd3439 100644 --- a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java @@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public class SmokeTestRankEvalWithMustacheYAMLTestSuiteIT extends ESClientYamlSu } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } From 6f6b2933b128eb7670f029bb897cd0c6a148f807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 16 Feb 2017 11:02:02 +0100 Subject: [PATCH 091/297] Fixing compile issues after merging in master --- .../index/rankeval/RatedSearchHit.java | 3 +-- .../rankeval/DiscountedCumulativeGainTests.java | 14 +++++++------- .../index/rankeval/PrecisionTests.java | 13 ++++++------- .../index/rankeval/RatedSearchHitTests.java | 7 +++---- .../index/rankeval/ReciprocalRankTests.java | 7 +++---- 5 files changed, 20 insertions(+), 24 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java index 0378f273657..fb2a46548d8 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.internal.InternalSearchHit; import java.io.IOException; import java.util.Objects; @@ -43,7 +42,7 @@ public class RatedSearchHit implements Writeable, ToXContent { } public RatedSearchHit(StreamInput in) throws IOException { - this(InternalSearchHit.readSearchHit(in), in.readBoolean() == true ? Optional.of(in.readVInt()) : Optional.empty()); + this(SearchHit.readSearchHit(in), in.readBoolean() == true ? Optional.of(in.readVInt()) : Optional.empty()); } @Override diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 209b8ca8fe3..7b97b1c2c2c 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -27,8 +27,8 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; -import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -57,10 +57,10 @@ public class DiscountedCumulativeGainTests extends ESTestCase { public void testDCGAt() { List rated = new ArrayList<>(); int[] relevanceRatings = new int[] { 3, 2, 3, 0, 1, 2 }; - InternalSearchHit[] hits = new InternalSearchHit[6]; + SearchHit[] hits = new SearchHit[6]; for (int i = 0; i < 6; i++) { rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); - hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); @@ -101,14 +101,14 @@ public class DiscountedCumulativeGainTests extends ESTestCase { public void testDCGAtSixMissingRatings() { List rated = new ArrayList<>(); Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1}; - InternalSearchHit[] hits = new InternalSearchHit[6]; + SearchHit[] hits = new SearchHit[6]; for (int i = 0; i < 6; i++) { if (i < relevanceRatings.length) { if (relevanceRatings[i] != null) { rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); } } - hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); @@ -161,9 +161,9 @@ public class DiscountedCumulativeGainTests extends ESTestCase { } } // only create four hits - InternalSearchHit[] hits = new InternalSearchHit[4]; + SearchHit[] hits = new SearchHit[4]; for (int i = 0; i < 4; i++) { - hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java index 77e6cfaff34..6e41960d823 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.Index; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; -import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -115,8 +114,8 @@ public class PrecisionTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); // add an unlabeled search hit SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test", "testtype"), 3); - searchHits[2] = new InternalSearchHit(2, "2", new Text("testtype"), Collections.emptyMap()); - ((InternalSearchHit)searchHits[2]).shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); + searchHits[2] = new SearchHit(2, "2", new Text("testtype"), Collections.emptyMap()); + searchHits[2].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); EvalQueryQuality evaluated = (new Precision()).evaluate("id", searchHits, rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); @@ -133,9 +132,9 @@ public class PrecisionTests extends ESTestCase { } public void testNoRatedDocs() throws Exception { - InternalSearchHit[] hits = new InternalSearchHit[5]; + SearchHit[] hits = new SearchHit[5]; for (int i = 0; i < 5; i++) { - hits[i] = new InternalSearchHit(i, i+"", new Text("type"), Collections.emptyMap()); + hits[i] = new SearchHit(i, i+"", new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); } EvalQueryQuality evaluated = (new Precision()).evaluate("id", hits, Collections.emptyList()); @@ -228,9 +227,9 @@ public class PrecisionTests extends ESTestCase { } private static SearchHit[] toSearchHits(List rated, String index, String type) { - InternalSearchHit[] hits = new InternalSearchHit[rated.size()]; + SearchHit[] hits = new SearchHit[rated.size()]; for (int i = 0; i < rated.size(); i++) { - hits[i] = new InternalSearchHit(i, i+"", new Text(type), Collections.emptyMap()); + hits[i] = new SearchHit(i, i+"", new Text(type), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java index 2e65d1fe846..4e5f5e2fc88 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.text.Text; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -32,7 +31,7 @@ public class RatedSearchHitTests extends ESTestCase { public static RatedSearchHit randomRatedSearchHit() { Optional rating = randomBoolean() ? Optional.empty() : Optional.of(randomIntBetween(0, 5)); - SearchHit searchHit = new InternalSearchHit(randomIntBetween(0, 10), randomAsciiOfLength(10), new Text(randomAsciiOfLength(10)), + SearchHit searchHit = new SearchHit(randomIntBetween(0, 10), randomAsciiOfLength(10), new Text(randomAsciiOfLength(10)), Collections.emptyMap()); RatedSearchHit ratedSearchHit = new RatedSearchHit(searchHit, rating); return ratedSearchHit; @@ -40,13 +39,13 @@ public class RatedSearchHitTests extends ESTestCase { private static RatedSearchHit mutateTestItem(RatedSearchHit original) { Optional rating = original.getRating(); - InternalSearchHit hit = (InternalSearchHit) original.getSearchHit(); + SearchHit hit = original.getSearchHit(); switch (randomIntBetween(0, 1)) { case 0: rating = rating.isPresent() ? Optional.of(rating.get() + 1) : Optional.of(randomInt(5)); break; case 1: - hit = new InternalSearchHit(hit.docId(), hit.getId() + randomAsciiOfLength(10), new Text(hit.getType()), + hit = new SearchHit(hit.docId(), hit.getId() + randomAsciiOfLength(10), new Text(hit.getType()), Collections.emptyMap()); break; default: diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index c66f7ba45aa..1f666f4c499 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionTests.Rating; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; -import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -140,13 +139,13 @@ public class ReciprocalRankTests extends ESTestCase { } /** - * Create InternalSearchHits for testing, starting from dociId 'from' up to docId 'to'. + * Create SearchHits for testing, starting from dociId 'from' up to docId 'to'. * The search hits index and type also need to be provided */ private static SearchHit[] createSearchHits(int from, int to, String index, String type) { - InternalSearchHit[] hits = new InternalSearchHit[to + 1 - from]; + SearchHit[] hits = new SearchHit[to + 1 - from]; for (int i = from; i <= to; i++) { - hits[i] = new InternalSearchHit(i, i+"", new Text(type), Collections.emptyMap()); + hits[i] = new SearchHit(i, i+"", new Text(type), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0)); } return hits; From ddae32705c27d0557d6377ec21ce6ea235c91108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 27 Feb 2017 11:42:46 +0100 Subject: [PATCH 092/297] Adapting build.gradle to changes on master --- modules/rank-eval/build.gradle | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle index cd4dd32c3d5..1be16253b27 100644 --- a/modules/rank-eval/build.gradle +++ b/modules/rank-eval/build.gradle @@ -22,9 +22,7 @@ esplugin { classname 'org.elasticsearch.index.rankeval.RankEvalPlugin' } -integTest { - cluster { - setting 'script.inline', 'true' - setting 'script.stored', 'true' - } +integTestCluster { + setting 'script.inline', 'true' + setting 'script.stored', 'true' } From f5388e579974073134867e79190c9b6da1e1cce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 14 Mar 2017 12:21:28 -0700 Subject: [PATCH 093/297] Adapting rank_eval integration tests --- .../test/rank_eval/10_basic.yaml | 10 ++-- .../rest-api-spec/test/rank_eval/20_dcg.yaml | 60 ++++++++++--------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index a0170f9722a..9c683bc474c 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -65,20 +65,20 @@ - match: { rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}} - length: { rank_eval.details.amsterdam_query.hits: 3} - - match: { rank_eval.details.amsterdam_query.hits.0.hit: {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "_score" : 0.44839138}} + - match: { rank_eval.details.amsterdam_query.hits.0.hit._id: "doc2"} - match: { rank_eval.details.amsterdam_query.hits.0.rating: 1} - - match: { rank_eval.details.amsterdam_query.hits.1.hit: {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "_score" : 0.44839138}} + - match: { rank_eval.details.amsterdam_query.hits.1.hit._id: "doc3"} - match: { rank_eval.details.amsterdam_query.hits.1.rating: 1} - - match: { rank_eval.details.amsterdam_query.hits.2.hit: {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "_score" : 0.21492207}} + - match: { rank_eval.details.amsterdam_query.hits.2.hit._id: "doc4"} - is_false: rank_eval.details.amsterdam_query.hits.2.rating - match: { rank_eval.details.berlin_query.quality_level: 1.0} - match: { rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} - match: { rank_eval.details.berlin_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}} - length: { rank_eval.details.berlin_query.hits: 2} - - match: { rank_eval.details.berlin_query.hits.0.hit: { "_index" : "foo", "_type" : "bar", "_id" : "doc1", "_score" : 0.87138504}} + - match: { rank_eval.details.berlin_query.hits.0.hit._id: "doc1" } - match: { rank_eval.details.berlin_query.hits.0.rating: 1} - - match: { rank_eval.details.berlin_query.hits.1.hit: { "_index" : "foo", "_type" : "bar", "_id" : "doc4", "_score" : 0.41767058}} + - match: { rank_eval.details.berlin_query.hits.1.hit._id: "doc4" } - is_false: rank_eval.details.berlin_query.hits.1.rating --- diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml index 2f784790ad5..580f21c1441 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml @@ -21,19 +21,21 @@ type: bar id: doc3 body: { "bar": 3 } - + - do: index: index: foo type: bar id: doc4 body: { "bar": 4 } + - do: index: index: foo type: bar id: doc5 body: { "bar": 5 } + - do: index: index: foo @@ -95,34 +97,34 @@ # if we mix both, we should get the average - do: - rank_eval: - body: { - "requests" : [ - { - "id": "dcg_query", - "request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] }, - "ratings": [ - {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] - }, - { - "id": "dcg_query_reverse", - "request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] }, - "ratings": [ - {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] - }, - ], - "metric" : { "dcg": { }} - } + rank_eval: + body: { + "requests" : [ + { + "id": "dcg_query", + "request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] }, + "ratings": [ + {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] + }, + { + "id": "dcg_query_reverse", + "request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] }, + "ratings": [ + {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, + {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] + }, + ], + "metric" : { "dcg": { }} + } - match: {rank_eval.quality_level: 12.073969010408984} - match: {rank_eval.details.dcg_query.quality_level: 13.84826362927298} From 4a75ede20818fe1864f1b5fb4bc96eb1ee757843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 23 Mar 2017 20:11:33 +0100 Subject: [PATCH 094/297] Reformatting source to fit 100 character line length restriction --- .../transport/TransportProxyClient.java | 1 - .../rankeval/DiscountedCumulativeGain.java | 41 ++-- .../index/rankeval/Precision.java | 50 +++-- .../index/rankeval/RankEvalAction.java | 3 +- .../index/rankeval/RankEvalPlugin.java | 27 ++- .../index/rankeval/RankEvalRequest.java | 9 +- .../rankeval/RankEvalRequestBuilder.java | 13 +- .../index/rankeval/RankEvalResponse.java | 39 ++-- .../index/rankeval/RankEvalSpec.java | 37 ++-- .../rankeval/RankedListQualityMetric.java | 51 +++-- .../index/rankeval/RatedRequest.java | 96 ++++----- .../index/rankeval/RatedSearchHit.java | 19 +- .../index/rankeval/ReciprocalRank.java | 35 ++-- .../index/rankeval/RestRankEvalAction.java | 15 +- .../rankeval/TransportRankEvalAction.java | 72 ++++--- .../DiscountedCumulativeGainTests.java | 158 ++++++++------- .../index/rankeval/EvalQueryQualityTests.java | 9 +- .../index/rankeval/PrecisionTests.java | 66 +++--- .../index/rankeval/RankEvalRequestIT.java | 61 +++--- .../index/rankeval/RankEvalResponseTests.java | 15 +- .../index/rankeval/RankEvalSpecTests.java | 116 ++++++----- .../index/rankeval/RankEvalTestHelper.java | 36 ++-- .../index/rankeval/RatedDocumentTests.java | 12 +- .../index/rankeval/RatedRequestsTests.java | 190 ++++++++++-------- .../index/rankeval/RatedSearchHitTests.java | 11 +- .../index/rankeval/ReciprocalRankTests.java | 45 +++-- 26 files changed, 719 insertions(+), 508 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java index c7a1f294324..5436bef172a 100644 --- a/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java +++ b/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java @@ -29,7 +29,6 @@ import org.elasticsearch.action.TransportActionNodeProxy; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.transport.TransportService; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java index 27b4fc5fda1..6f843f92461 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java @@ -41,7 +41,10 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { /** If set to true, the dcg will be normalized (ndcg) */ private boolean normalize; - /** If set to, this will be the rating for docs the user hasn't supplied an explicit rating for */ + /** + * If set to, this will be the rating for docs the user hasn't supplied an + * explicit rating for + */ private Integer unknownDocRating; public static final String NAME = "dcg"; @@ -51,11 +54,14 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { } /** - * @param normalize If set to true, dcg will be normalized (ndcg) - * See https://en.wikipedia.org/wiki/Discounted_cumulative_gain - * @param unknownDocRating the rating for docs the user hasn't supplied an explicit rating for - * */ - public DiscountedCumulativeGain( boolean normalize, Integer unknownDocRating) { + * @param normalize + * If set to true, dcg will be normalized (ndcg) See + * https://en.wikipedia.org/wiki/Discounted_cumulative_gain + * @param unknownDocRating + * the rating for docs the user hasn't supplied an explicit + * rating for + */ + public DiscountedCumulativeGain(boolean normalize, Integer unknownDocRating) { this.normalize = normalize; this.unknownDocRating = unknownDocRating; } @@ -105,20 +111,25 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { } @Override - public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { - List allRatings = ratedDocs.stream().mapToInt(RatedDocument::getRating).boxed().collect(Collectors.toList()); + public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, + List ratedDocs) { + List allRatings = ratedDocs.stream().mapToInt(RatedDocument::getRating).boxed() + .collect(Collectors.toList()); List ratedHits = joinHitsWithRatings(hits, ratedDocs); List ratingsInSearchHits = new ArrayList<>(ratedHits.size()); for (RatedSearchHit hit : ratedHits) { - // unknownDocRating might be null, which means it will be unrated docs are ignored in the dcg calculation - // we still need to add them as a placeholder so the rank of the subsequent ratings is correct + // unknownDocRating might be null, which means it will be unrated + // docs are ignored in the dcg calculation + // we still need to add them as a placeholder so the rank of the + // subsequent ratings is correct ratingsInSearchHits.add(hit.getRating().orElse(unknownDocRating)); } double dcg = computeDCG(ratingsInSearchHits); if (normalize) { Collections.sort(allRatings, Comparator.nullsLast(Collections.reverseOrder())); - double idcg = computeDCG(allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size()))); + double idcg = computeDCG( + allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size()))); dcg = dcg / idcg; } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, dcg); @@ -140,8 +151,8 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { private static final ParseField NORMALIZE_FIELD = new ParseField("normalize"); private static final ParseField UNKNOWN_DOC_RATING_FIELD = new ParseField("unknown_doc_rating"); - private static final ObjectParser PARSER = - new ObjectParser<>("dcg_at", () -> new DiscountedCumulativeGain()); + private static final ObjectParser PARSER = new ObjectParser<>( + "dcg_at", () -> new DiscountedCumulativeGain()); static { PARSER.declareBoolean(DiscountedCumulativeGain::setNormalize, NORMALIZE_FIELD); @@ -174,8 +185,8 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { return false; } DiscountedCumulativeGain other = (DiscountedCumulativeGain) obj; - return Objects.equals(normalize, other.normalize) && - Objects.equals(unknownDocRating, other.unknownDocRating); + return Objects.equals(normalize, other.normalize) + && Objects.equals(unknownDocRating, other.unknownDocRating); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java index 40923fe1a0a..72055b01137 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java @@ -46,14 +46,17 @@ public class Precision implements RankedListQualityMetric { public static final String NAME = "precision"; - private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); + private static final ParseField RELEVANT_RATING_FIELD = new ParseField( + "relevant_rating_threshold"); private static final ParseField IGNORE_UNLABELED_FIELD = new ParseField("ignore_unlabeled"); - private static final ObjectParser PARSER = new ObjectParser<>(NAME, Precision::new); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, + Precision::new); /** * This setting controls how unlabeled documents in the search hits are * treated. Set to 'true', unlabeled documents are ignored and neither count - * as true or false positives. Set to 'false', they are treated as false positives. + * as true or false positives. Set to 'false', they are treated as false + * positives. */ private boolean ignoreUnlabeled = false; @@ -86,33 +89,35 @@ public class Precision implements RankedListQualityMetric { } /** - * Sets the rating threshold above which ratings are considered to be "relevant" for this metric. - * */ + * Sets the rating threshold above which ratings are considered to be + * "relevant" for this metric. + */ public void setRelevantRatingThreshhold(int threshold) { if (threshold < 0) { - throw new IllegalArgumentException("Relevant rating threshold for precision must be positive integer."); + throw new IllegalArgumentException( + "Relevant rating threshold for precision must be positive integer."); } this.relevantRatingThreshhold = threshold; } /** - * Return the rating threshold above which ratings are considered to be "relevant" for this metric. - * Defaults to 1. - * */ + * Return the rating threshold above which ratings are considered to be + * "relevant" for this metric. Defaults to 1. + */ public int getRelevantRatingThreshold() { - return relevantRatingThreshhold ; + return relevantRatingThreshhold; } /** * Sets the 'ìgnore_unlabeled' parameter - * */ + */ public void setIgnoreUnlabeled(boolean ignoreUnlabeled) { this.ignoreUnlabeled = ignoreUnlabeled; } /** * Gets the 'ìgnore_unlabeled' parameter - * */ + */ public boolean getIgnoreUnlabeled() { return ignoreUnlabeled; } @@ -121,11 +126,14 @@ public class Precision implements RankedListQualityMetric { return PARSER.apply(parser, null); } - /** Compute precisionAtN based on provided relevant document IDs. + /** + * Compute precisionAtN based on provided relevant document IDs. + * * @return precision at n for above {@link SearchResult} list. **/ @Override - public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { + public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, + List ratedDocs) { int truePositives = 0; int falsePositives = 0; List ratedSearchHits = joinHitsWithRatings(hits, ratedDocs); @@ -146,7 +154,8 @@ public class Precision implements RankedListQualityMetric { precision = (double) truePositives / (truePositives + falsePositives); } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision); - evalQueryQuality.addMetricDetails(new Precision.Breakdown(truePositives, truePositives + falsePositives)); + evalQueryQuality.addMetricDetails( + new Precision.Breakdown(truePositives, truePositives + falsePositives)); evalQueryQuality.addHitsAndRatings(ratedSearchHits); return evalQueryQuality; } @@ -171,8 +180,8 @@ public class Precision implements RankedListQualityMetric { return false; } Precision other = (Precision) obj; - return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold) && - Objects.equals(ignoreUnlabeled, other.ignoreUnlabeled); + return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold) + && Objects.equals(ignoreUnlabeled, other.ignoreUnlabeled); } @Override @@ -198,7 +207,8 @@ public class Precision implements RankedListQualityMetric { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + public XContentBuilder toXContent(XContentBuilder builder, Params params) + throws IOException { builder.field(RELEVANT_DOCS_RETRIEVED_FIELD, relevantRetrieved); builder.field(DOCS_RETRIEVED_FIELD, retrieved); return builder; @@ -232,8 +242,8 @@ public class Precision implements RankedListQualityMetric { return false; } Precision.Breakdown other = (Precision.Breakdown) obj; - return Objects.equals(relevantRetrieved, other.relevantRetrieved) && - Objects.equals(retrieved, other.retrieved); + return Objects.equals(relevantRetrieved, other.relevantRetrieved) + && Objects.equals(retrieved, other.retrieved); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java index b8ff4b08563..b2e4aeb8777 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java @@ -25,7 +25,8 @@ import org.elasticsearch.client.ElasticsearchClient; /** * Action used to start precision at qa evaluations. **/ -public class RankEvalAction extends Action { +public class RankEvalAction + extends Action { public static final RankEvalAction INSTANCE = new RankEvalAction(); public static final String NAME = "indices:data/read/quality"; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index ca2654fb943..7bc291e35da 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -43,32 +43,37 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin { @Override public List> getActions() { - return Arrays.asList(new ActionHandler<>(RankEvalAction.INSTANCE, TransportRankEvalAction.class)); + return Arrays.asList( + new ActionHandler<>(RankEvalAction.INSTANCE, TransportRankEvalAction.class)); } - @Override - public List getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings, - IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, + public List getRestHandlers(Settings settings, RestController restController, + ClusterSettings clusterSettings, IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster) { return Arrays.asList(new RestRankEvalAction(settings, restController)); } /** - * Returns parsers for {@link NamedWriteable} this plugin will use over the transport protocol. + * Returns parsers for {@link NamedWriteable} this plugin will use over the + * transport protocol. + * * @see NamedWriteableRegistry */ @Override public List getNamedWriteables() { List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, - DiscountedCumulativeGain::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, + Precision.NAME, Precision::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, + ReciprocalRank.NAME, ReciprocalRank::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, + DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, Precision.NAME, Precision.Breakdown::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, ReciprocalRank.NAME, - ReciprocalRank.Breakdown::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, + ReciprocalRank.NAME, ReciprocalRank.Breakdown::new)); return namedWriteables; } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java index 2f1cdf44f86..2d75bc88671 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java @@ -26,11 +26,11 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; - /** - * Instances of this class represent a complete precision at request. They encode a precision task including search intents and search - * specifications to be executed subsequently. - * */ + * Instances of this class represent a complete precision at request. They + * encode a precision task including search intents and search specifications to + * be executed subsequently. + */ public class RankEvalRequest extends ActionRequest { /** The request data to use for evaluation. */ @@ -62,7 +62,6 @@ public class RankEvalRequest extends ActionRequest { this.task = task; } - @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java index 063bec9d8f7..77306259fe4 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java @@ -23,21 +23,24 @@ import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -public class RankEvalRequestBuilder extends ActionRequestBuilder { - - public RankEvalRequestBuilder(ElasticsearchClient client, Action action, +public class RankEvalRequestBuilder + extends ActionRequestBuilder { + + public RankEvalRequestBuilder(ElasticsearchClient client, + Action action, RankEvalRequest request) { super(client, action, request); } + @Override public RankEvalRequest request() { return request; } - + public void setRankEvalSpec(RankEvalSpec spec) { this.request.setRankEvalSpec(spec); } - + public RankEvalSpec getRankEvalSpec() { return this.request.getRankEvalSpec(); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 5364fb87cdc..b5904422f04 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -32,28 +32,41 @@ import java.util.HashMap; import java.util.Map; /** - * For each qa specification identified by its id this response returns the respective - * averaged precisionAnN value. + * For each qa specification identified by its id this response returns the + * respective averaged precisionAnN value. * - * In addition for each query the document ids that haven't been found annotated is returned as well. + * In addition for each query the document ids that haven't been found annotated + * is returned as well. * - * Documents of unknown quality - i.e. those that haven't been supplied in the set of annotated documents but have been returned - * by the search are not taken into consideration when computing precision at n - they are ignored. + * Documents of unknown quality - i.e. those that haven't been supplied in the + * set of annotated documents but have been returned by the search are not taken + * into consideration when computing precision at n - they are ignored. * **/ -//TODO instead of just returning averages over complete results, think of other statistics, micro avg, macro avg, partial results +// TODO instead of just returning averages over complete results, think of other +// statistics, micro avg, macro avg, partial results public class RankEvalResponse extends ActionResponse implements ToXContentObject { - /**Average precision observed when issuing query intents with this specification.*/ + /** + * Average precision observed when issuing query intents with this + * specification. + */ private double qualityLevel; - /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ + /** + * Mapping from intent id to all documents seen for this intent that were + * not annotated. + */ private Map details; - /**Mapping from intent id to potential exceptions that were thrown on query execution.*/ + /** + * Mapping from intent id to potential exceptions that were thrown on query + * execution. + */ private Map failures; public RankEvalResponse() { } - public RankEvalResponse(double qualityLevel, Map partialResults, Map failures) { + public RankEvalResponse(double qualityLevel, Map partialResults, + Map failures) { this.qualityLevel = qualityLevel; this.details = partialResults; this.failures = failures; @@ -73,7 +86,8 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject @Override public String toString() { - return "RankEvalResponse, quality: " + qualityLevel + ", partial results: " + details + ", number of failures: " + failures.size(); + return "RankEvalResponse, quality: " + qualityLevel + ", partial results: " + details + + ", number of failures: " + failures.size(); } @Override @@ -124,7 +138,8 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject builder.startObject("failures"); for (String key : failures.keySet()) { builder.startObject(key); - ElasticsearchException.generateFailureXContent(builder, params, failures.get(key), true); + ElasticsearchException.generateFailureXContent(builder, params, failures.get(key), + true); builder.endObject(); } builder.endObject(); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index e45ba14eda6..5c7a713f23b 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -39,14 +39,19 @@ import java.util.Map.Entry; import java.util.Objects; /** - * This class defines a ranking evaluation task including an id, a collection of queries to evaluate and the evaluation metric. + * This class defines a ranking evaluation task including an id, a collection of + * queries to evaluate and the evaluation metric. * - * Each QA run is based on a set of queries to send to the index and multiple QA specifications that define how to translate the query - * intents into elastic search queries. - * */ + * Each QA run is based on a set of queries to send to the index and multiple QA + * specifications that define how to translate the query intents into elastic + * search queries. + */ public class RankEvalSpec extends ToXContentToBytes implements Writeable { - /** Collection of query specifications, that is e.g. search request templates to use for query translation. */ + /** + * Collection of query specifications, that is e.g. search request templates + * to use for query translation. + */ private Collection ratedRequests = new ArrayList<>(); /** Definition of the quality metric, e.g. precision at N */ private RankedListQualityMetric metric; @@ -57,10 +62,12 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { /** optional: Templates to base test requests on */ private Map templates = new HashMap<>(); - public RankEvalSpec(Collection ratedRequests, RankedListQualityMetric metric, Collection templates) { + public RankEvalSpec(Collection ratedRequests, RankedListQualityMetric metric, + Collection templates) { if (ratedRequests == null || ratedRequests.size() < 1) { throw new IllegalStateException( - "Cannot evaluate ranking if no search requests with rated results are provided. Seen: " + ratedRequests); + "Cannot evaluate ranking if no search requests with rated results are provided." + + " Seen: " + ratedRequests); } if (metric == null) { throw new IllegalStateException( @@ -70,8 +77,8 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { for (RatedRequest request : ratedRequests) { if (request.getTestRequest() == null) { throw new IllegalStateException( - "Cannot evaluate ranking if neither template nor test request is provided. Seen for request id: " - + request.getId()); + "Cannot evaluate ranking if neither template nor test request is " + + "provided. Seen for request id: " + request.getId()); } } } @@ -147,11 +154,13 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { private static final ParseField TEMPLATES_FIELD = new ParseField("templates"); private static final ParseField METRIC_FIELD = new ParseField("metric"); private static final ParseField REQUESTS_FIELD = new ParseField("requests"); - private static final ParseField MAX_CONCURRENT_SEARCHES_FIELD = new ParseField("max_concurrent_searches"); + private static final ParseField MAX_CONCURRENT_SEARCHES_FIELD + = new ParseField("max_concurrent_searches"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rank_eval", - a -> new RankEvalSpec((Collection) a[0], (RankedListQualityMetric) a[1], (Collection) a[2])); + a -> new RankEvalSpec((Collection) a[0], + (RankedListQualityMetric) a[1], (Collection) a[2])); static { PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> { @@ -187,7 +196,8 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("script_with_id", a -> new ScriptWithId((String) a[0], (Script) a[1])); + new ConstructingObjectParser<>("script_with_id", + a -> new ScriptWithId((String) a[0], (Script) a[1])); public static ScriptWithId fromXContent(XContentParser parser) { return PARSER.apply(parser, null); @@ -199,7 +209,8 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { try { return Script.parse(p, "mustache"); } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", + ex); } }, TEMPLATE_FIELD); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index ea42f32ff07..15d4eb03064 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -35,29 +35,38 @@ import java.util.Optional; import java.util.stream.Collectors; /** - * Classes implementing this interface provide a means to compute the quality of a result list - * returned by some search. + * Classes implementing this interface provide a means to compute the quality of + * a result list returned by some search. * - * RelevancyLevel specifies the type of object determining the relevancy level of some known docid. - * */ + * RelevancyLevel specifies the type of object determining the relevancy level + * of some known docid. + */ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { /** - * Returns a single metric representing the ranking quality of a set of returned documents - * wrt. to a set of document Ids labeled as relevant for this search. + * Returns a single metric representing the ranking quality of a set of + * returned documents wrt. to a set of document Ids labeled as relevant for + * this search. * - * @param taskId the id of the query for which the ranking is currently evaluated - * @param hits the result hits as returned by a search request - * @param ratedDocs the documents that were ranked by human annotators for this query case - * @return some metric representing the quality of the result hit list wrt. to relevant doc ids. - * */ + * @param taskId + * the id of the query for which the ranking is currently + * evaluated + * @param hits + * the result hits as returned by a search request + * @param ratedDocs + * the documents that were ranked by human annotators for this + * query case + * @return some metric representing the quality of the result hit list wrt. + * to relevant doc ids. + */ EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs); static RankedListQualityMetric fromXContent(XContentParser parser) throws IOException { RankedListQualityMetric rc; Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { - throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name"); + throw new ParsingException(parser.getTokenLocation(), + "[_na] missing required metric name"); } String metricName = parser.currentName(); @@ -73,7 +82,8 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { rc = DiscountedCumulativeGain.fromXContent(parser); break; default: - throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName); + throw new ParsingException(parser.getTokenLocation(), + "[_na] unknown query metric name [{}]", metricName); } if (parser.currentToken() == XContentParser.Token.END_OBJECT) { // if we are at END_OBJECT, move to the next one... @@ -82,8 +92,9 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { return rc; } - static List joinHitsWithRatings(SearchHit[] hits, List ratedDocs) { - // join hits with rated documents + static List joinHitsWithRatings(SearchHit[] hits, + List ratedDocs) { + // join hits with rated documents Map ratedDocumentMap = ratedDocs.stream() .collect(Collectors.toMap(RatedDocument::getKey, item -> item)); List ratedSearchHits = new ArrayList<>(hits.length); @@ -103,12 +114,14 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { // join hits with rated documents List unknownDocs = ratedHits.stream() .filter(hit -> hit.getRating().isPresent() == false) - .map(hit -> new DocumentKey(hit.getSearchHit().getIndex(), hit.getSearchHit().getType(), hit.getSearchHit().getId())) + .map(hit -> new DocumentKey(hit.getSearchHit().getIndex(), + hit.getSearchHit().getType(), hit.getSearchHit().getId())) .collect(Collectors.toList()); - return unknownDocs; - } + return unknownDocs; + } default double combine(Collection partialResults) { - return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size(); + return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() + / partialResults.size(); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 395b77ef801..f673991a490 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -43,36 +43,47 @@ import java.util.Objects; import java.util.Set; /** - * Defines a QA specification: All end user supplied query intents will be mapped to the search request specified in this search request - * template and executed against the targetIndex given. Any filters that should be applied in the target system can be specified as well. + * Defines a QA specification: All end user supplied query intents will be + * mapped to the search request specified in this search request template and + * executed against the targetIndex given. Any filters that should be applied in + * the target system can be specified as well. * - * The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery. - * */ + * The resulting document lists can then be compared against what was specified + * in the set of rated documents as part of a QAQuery. + */ @SuppressWarnings("unchecked") public class RatedRequest extends ToXContentToBytes implements Writeable { private String id; private List indices = new ArrayList<>(); private List types = new ArrayList<>(); private List summaryFields = new ArrayList<>(); - /** Collection of rated queries for this query QA specification.*/ + /** Collection of rated queries for this query QA specification. */ private List ratedDocs = new ArrayList<>(); - /** Search request to execute for this rated request, can be null, if template and corresponding params are supplied. */ + /** + * Search request to execute for this rated request, can be null, if + * template and corresponding params are supplied. + */ @Nullable private SearchSourceBuilder testRequest; - /** Map of parameters to use for filling a query template, can be used instead of providing testRequest. */ + /** + * Map of parameters to use for filling a query template, can be used + * instead of providing testRequest. + */ private Map params = new HashMap<>(); @Nullable private String templateId; - public RatedRequest( - String id, List ratedDocs, SearchSourceBuilder testRequest, Map params, String templateId) { + public RatedRequest(String id, List ratedDocs, SearchSourceBuilder testRequest, + Map params, String templateId) { if (params != null && (params.size() > 0 && testRequest != null)) { throw new IllegalArgumentException( - "Ambiguous rated request: Set both, verbatim test request and test request template parameters."); + "Ambiguous rated request: Set both, verbatim test request and test request " + + "template parameters."); } if (templateId != null && testRequest != null) { throw new IllegalArgumentException( - "Ambiguous rated request: Set both, verbatim test request and test request template parameters."); + "Ambiguous rated request: Set both, verbatim test request and test request " + + "template parameters."); } if ((params == null || params.size() < 1) && testRequest == null) { throw new IllegalArgumentException( @@ -80,13 +91,15 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { } if ((params != null && params.size() > 0) && templateId == null) { throw new IllegalArgumentException( - "If template parameters are supplied need to set id of template to apply them to too."); + "If template parameters are supplied need to set id of template to apply " + + "them to too."); } // No documents with same _index/_type/id allowed. Set docKeys = new HashSet<>(); for (RatedDocument doc : ratedDocs) { if (docKeys.add(doc.getKey()) == false) { - String docKeyToString = doc.getKey().toString().replaceAll("\n", "").replaceAll(" ", " "); + String docKeyToString = doc.getKey().toString().replaceAll("\n", "") + .replaceAll(" ", " "); throw new IllegalArgumentException( "Found duplicate rated document key [" + docKeyToString + "]"); } @@ -101,7 +114,8 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { this.templateId = templateId; } - public RatedRequest(String id, List ratedDocs, Map params, String templateId) { + public RatedRequest(String id, List ratedDocs, Map params, + String templateId) { this(id, ratedDocs, null, params, templateId); } @@ -200,7 +214,10 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { return this.templateId; } - /** Returns a list of fields that are included in the docs summary of matched documents. */ + /** + * Returns a list of fields that are included in the docs summary of matched + * documents. + */ public List getSummaryFields() { return summaryFields; } @@ -220,13 +237,14 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { private static final ParseField TEMPLATE_ID_FIELD = new ParseField("template_id"); private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("requests", a -> new RatedRequest( - (String) a[0], (List) a[1], (SearchSourceBuilder) a[2], (Map) a[3], (String) a[4])); + new ConstructingObjectParser<>("requests", + a -> new RatedRequest((String) a[0], (List) a[1], + (SearchSourceBuilder) a[2], (Map) a[3], (String) a[4])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD); PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> { - return RatedDocument.fromXContent(p); + return RatedDocument.fromXContent(p); }, RATINGS_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { try { @@ -234,7 +252,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing request", ex); } - } , REQUEST_FIELD); + }, REQUEST_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { try { return (Map) p.map(); @@ -249,23 +267,12 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { /** * Parses {@link RatedRequest} from rest representation: * - * Example: - * { - * "id": "coffee_query", - * "request": { - * "query": { - * "bool": { - * "must": [ - * {"match": {"beverage": "coffee"}}, - * {"term": {"browser": {"value": "safari"}}}, - * {"term": {"time_of_day": {"value": "morning","boost": 2}}}, - * {"term": {"ip_location": {"value": "ams","boost": 10}}}]} - * }, - * "size": 10 - * }, - * "summary_fields" : ["body"], - * "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ] - * } + * Example: { "id": "coffee_query", "request": { "query": { "bool": { + * "must": [ {"match": {"beverage": "coffee"}}, {"term": {"browser": + * {"value": "safari"}}}, {"term": {"time_of_day": {"value": + * "morning","boost": 2}}}, {"term": {"ip_location": {"value": + * "ams","boost": 10}}}]} }, "size": 10 }, "summary_fields" : ["body"], + * "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ] } */ public static RatedRequest fromXContent(XContentParser parser) { return PARSER.apply(parser, new QueryParseContext(parser)); @@ -313,18 +320,17 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { RatedRequest other = (RatedRequest) obj; - return Objects.equals(id, other.id) && - Objects.equals(testRequest, other.testRequest) && - Objects.equals(indices, other.indices) && - Objects.equals(types, other.types) && - Objects.equals(summaryFields, other.summaryFields) && - Objects.equals(ratedDocs, other.ratedDocs) && - Objects.equals(params, other.params) && - Objects.equals(templateId, other.templateId); + return Objects.equals(id, other.id) && Objects.equals(testRequest, other.testRequest) + && Objects.equals(indices, other.indices) && Objects.equals(types, other.types) + && Objects.equals(summaryFields, other.summaryFields) + && Objects.equals(ratedDocs, other.ratedDocs) + && Objects.equals(params, other.params) + && Objects.equals(templateId, other.templateId); } @Override public final int hashCode() { - return Objects.hash(id, testRequest, indices, types, summaryFields, ratedDocs, params, templateId); + return Objects.hash(id, testRequest, indices, types, summaryFields, ratedDocs, params, + templateId); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java index fb2a46548d8..9f5838e070f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java @@ -42,7 +42,8 @@ public class RatedSearchHit implements Writeable, ToXContent { } public RatedSearchHit(StreamInput in) throws IOException { - this(SearchHit.readSearchHit(in), in.readBoolean() == true ? Optional.of(in.readVInt()) : Optional.empty()); + this(SearchHit.readSearchHit(in), + in.readBoolean() == true ? Optional.of(in.readVInt()) : Optional.empty()); } @Override @@ -63,7 +64,8 @@ public class RatedSearchHit implements Writeable, ToXContent { } @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) + throws IOException { builder.startObject(); builder.field("hit", (ToXContent) searchHit); builder.field("rating", rating.orElse(null)); @@ -80,7 +82,8 @@ public class RatedSearchHit implements Writeable, ToXContent { return false; } RatedSearchHit other = (RatedSearchHit) obj; - // NORELEASE this is a workaround because InternalSearchHit does not properly implement equals()/hashCode(), so we compare their + // NORELEASE this is a workaround because InternalSearchHit does not + // properly implement equals()/hashCode(), so we compare their // xcontent XContentBuilder builder; String hitAsXContent; @@ -89,17 +92,19 @@ public class RatedSearchHit implements Writeable, ToXContent { builder = XContentFactory.jsonBuilder(); hitAsXContent = searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS).string(); builder = XContentFactory.jsonBuilder(); - otherHitAsXContent = other.searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS).string(); + otherHitAsXContent = other.searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS) + .string(); } catch (IOException e) { throw new RuntimeException(e); } - return Objects.equals(rating, other.rating) && - Objects.equals(hitAsXContent, otherHitAsXContent); + return Objects.equals(rating, other.rating) + && Objects.equals(hitAsXContent, otherHitAsXContent); } @Override public final int hashCode() { - // NORELEASE for this to work requires InternalSearchHit to properly implement equals()/hashCode() + // NORELEASE for this to work requires InternalSearchHit to properly + // implement equals()/hashCode() XContentBuilder builder; String hitAsXContent; try { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 2c4021ce2fc..17699de1d40 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -37,10 +37,10 @@ import javax.naming.directory.SearchResult; import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; /** - * Evaluate reciprocal rank. - * By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the reciprocal rank - * calculation. This value can be changes using the "relevant_rating_threshold" parameter. - * */ + * Evaluate reciprocal rank. By default documents with a rating equal or bigger + * than 1 are considered to be "relevant" for the reciprocal rank calculation. + * This value can be changes using the "relevant_rating_threshold" parameter. + */ public class ReciprocalRank implements RankedListQualityMetric { public static final String NAME = "reciprocal_rank"; @@ -65,30 +65,34 @@ public class ReciprocalRank implements RankedListQualityMetric { } /** - * Sets the rating threshold above which ratings are considered to be "relevant" for this metric. - * */ + * Sets the rating threshold above which ratings are considered to be + * "relevant" for this metric. + */ public void setRelevantRatingThreshhold(int threshold) { if (threshold < 0) { - throw new IllegalArgumentException("Relevant rating threshold for precision must be positive integer."); + throw new IllegalArgumentException( + "Relevant rating threshold for precision must be positive integer."); } this.relevantRatingThreshhold = threshold; } /** - * Return the rating threshold above which ratings are considered to be "relevant" for this metric. - * Defaults to 1. - * */ + * Return the rating threshold above which ratings are considered to be + * "relevant" for this metric. Defaults to 1. + */ public int getRelevantRatingThreshold() { - return relevantRatingThreshhold ; + return relevantRatingThreshhold; } /** * Compute ReciprocalRank based on provided relevant document IDs. + * * @return reciprocal Rank for above {@link SearchResult} list. **/ @Override - public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { + public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, + List ratedDocs) { List ratedHits = joinHitsWithRatings(hits, ratedDocs); int firstRelevant = -1; int rank = 1; @@ -115,7 +119,8 @@ public class ReciprocalRank implements RankedListQualityMetric { out.writeVInt(relevantRatingThreshhold); } - private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); + private static final ParseField RELEVANT_RATING_FIELD = new ParseField( + "relevant_rating_threshold"); private static final ObjectParser PARSER = new ObjectParser<>( "reciprocal_rank", () -> new ReciprocalRank()); @@ -167,7 +172,8 @@ public class ReciprocalRank implements RankedListQualityMetric { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + public XContentBuilder toXContent(XContentBuilder builder, Params params) + throws IOException { builder.field("first_relevant", firstRelevantRank); return builder; } @@ -185,6 +191,7 @@ public class ReciprocalRank implements RankedListQualityMetric { public int getFirstRelevantRank() { return firstRelevantRank; } + @Override public final boolean equals(Object obj) { if (this == obj) { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 3acba41e54f..1d721d69c09 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -163,7 +163,8 @@ public class RestRankEvalAction extends BaseRestHandler { } @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) + throws IOException { RankEvalRequest rankEvalRequest = new RankEvalRequest(); try (XContentParser parser = request.contentOrSourceParamParser()) { parseRankEvalRequest(rankEvalRequest, request, parser); @@ -172,15 +173,19 @@ public class RestRankEvalAction extends BaseRestHandler { new RestToXContentListener(channel)); } - private static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, XContentParser parser) { - List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); - List types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type"))); + private static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, + XContentParser parser) { + List indices = Arrays + .asList(Strings.splitStringByCommaToArray(request.param("index"))); + List types = Arrays + .asList(Strings.splitStringByCommaToArray(request.param("type"))); RankEvalSpec spec = null; spec = RankEvalSpec.parse(parser); for (RatedRequest specification : spec.getRatedRequests()) { specification.setIndices(indices); specification.setTypes(types); - }; + } + ; rankEvalRequest.setRankEvalSpec(spec); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index fafda7bb704..eb775312b86 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -56,27 +56,32 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.xcontent.XContentHelper.createParser; /** - * Instances of this class execute a collection of search intents (read: user supplied query parameters) against a set of - * possible search requests (read: search specifications, expressed as query/search request templates) and compares the result - * against a set of annotated documents per search intent. + * Instances of this class execute a collection of search intents (read: user + * supplied query parameters) against a set of possible search requests (read: + * search specifications, expressed as query/search request templates) and + * compares the result against a set of annotated documents per search intent. * - * If any documents are returned that haven't been annotated the document id of those is returned per search intent. + * If any documents are returned that haven't been annotated the document id of + * those is returned per search intent. * - * The resulting search quality is computed in terms of precision at n and returned for each search specification for the full - * set of search intents as averaged precision at n. - * */ -public class TransportRankEvalAction extends HandledTransportAction { + * The resulting search quality is computed in terms of precision at n and + * returned for each search specification for the full set of search intents as + * averaged precision at n. + */ +public class TransportRankEvalAction + extends HandledTransportAction { private Client client; private ScriptService scriptService; Queue taskQueue = new ConcurrentLinkedQueue<>(); private NamedXContentRegistry namedXContentRegistry; @Inject - public TransportRankEvalAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, Client client, TransportService transportService, - ScriptService scriptService, NamedXContentRegistry namedXContentRegistry) { - super(settings, RankEvalAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, - RankEvalRequest::new); + public TransportRankEvalAction(Settings settings, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + Client client, TransportService transportService, ScriptService scriptService, + NamedXContentRegistry namedXContentRegistry) { + super(settings, RankEvalAction.NAME, threadPool, transportService, actionFilters, + indexNameExpressionResolver, RankEvalRequest::new); this.scriptService = scriptService; this.namedXContentRegistry = namedXContentRegistry; this.client = client; @@ -88,26 +93,28 @@ public class TransportRankEvalAction extends HandledTransportAction ratedRequests = qualityTask.getRatedRequests(); AtomicInteger responseCounter = new AtomicInteger(ratedRequests.size()); - Map partialResults = new ConcurrentHashMap<>(ratedRequests.size()); + Map partialResults = new ConcurrentHashMap<>( + ratedRequests.size()); Map errors = new ConcurrentHashMap<>(ratedRequests.size()); Map scriptsWithoutParams = new HashMap<>(); for (Entry entry : qualityTask.getTemplates().entrySet()) { - scriptsWithoutParams.put( - entry.getKey(), - scriptService.compile(entry.getValue(), ScriptContext.Standard.SEARCH)); + scriptsWithoutParams.put(entry.getKey(), + scriptService.compile(entry.getValue(), ScriptContext.Standard.SEARCH)); } for (RatedRequest ratedRequest : ratedRequests) { - final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask.getMetric(), ratedRequest, - partialResults, errors, responseCounter); + final RankEvalActionListener searchListener = new RankEvalActionListener(listener, + qualityTask.getMetric(), ratedRequest, partialResults, errors, responseCounter); SearchSourceBuilder ratedSearchSource = ratedRequest.getTestRequest(); if (ratedSearchSource == null) { Map params = ratedRequest.getParams(); String templateId = ratedRequest.getTemplateId(); CompiledScript compiled = scriptsWithoutParams.get(templateId); - BytesReference resolvedRequest = (BytesReference) (scriptService.executable(compiled, params).run()); - try (XContentParser subParser = createParser(namedXContentRegistry, resolvedRequest, XContentType.JSON)) { + BytesReference resolvedRequest = (BytesReference) (scriptService + .executable(compiled, params).run()); + try (XContentParser subParser = createParser(namedXContentRegistry, resolvedRequest, + XContentType.JSON)) { QueryParseContext parseContext = new QueryParseContext(subParser); ratedSearchSource = SearchSourceBuilder.fromXContent(parseContext); } catch (IOException e) { @@ -118,7 +125,8 @@ public class TransportRankEvalAction extends HandledTransportAction listener, RankedListQualityMetric metric, RatedRequest specification, - Map details, Map errors, AtomicInteger responseCounter) { + public RankEvalActionListener(ActionListener listener, + RankedListQualityMetric metric, RatedRequest specification, + Map details, Map errors, + AtomicInteger responseCounter) { this.listener = listener; this.metric = metric; this.errors = errors; @@ -172,7 +184,8 @@ public class TransportRankEvalAction extends HandledTransportAction rated = new ArrayList<>(); - Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1}; + Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1 }; SearchHit[] hits = new SearchHit[6]; for (int i = 0; i < 6; i++) { if (i < relevanceRatings.length) { if (relevanceRatings[i] != null) { - rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); + rated.add(new RatedDocument("index", "type", Integer.toString(i), + relevanceRatings[i])); } } - hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), + Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); @@ -117,84 +122,85 @@ public class DiscountedCumulativeGainTests extends ESTestCase { assertEquals(2, filterUnknownDocuments(result.getHitsAndRatings()).size()); /** - * Check with normalization: to get the maximal possible dcg, sort documents by relevance in descending order + * Check with normalization: to get the maximal possible dcg, sort + * documents by relevance in descending order * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) - * ------------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0  | 7.0 - * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 - * 3 | 2 | 3.0 | 2.0  | 1.5 - * 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339 - * 5 | n.a | n.a | n.a.  | n.a. - * 6 | n.a | n.a | n.a  | n.a + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) + * - 1) / log_2(rank + 1) + * ---------------------------------------------------------------------------------------- + * 1 | 3 | 7.0 | 1.0  | 7.0 2 | 3 | 7.0 | + * 1.5849625007211563 | 4.416508275000202 3 | 2 | 3.0 | 2.0  | 1.5 4 | 1 + * | 1.0 | 2.321928094887362   | 0.43067655807339 5 | n.a | n.a | n.a.  + * | n.a. 6 | n.a | n.a | n.a  | n.a * * idcg = 13.347184833073591 (sum of last column) */ dcg.setNormalize(true); - assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); + assertEquals(12.779642067948913 / 13.347184833073591, + dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); } /** - * This tests that normalization works as expected when there are more rated documents than search hits - * because we restrict DCG to be calculated at the fourth position + * This tests that normalization works as expected when there are more rated + * documents than search hits because we restrict DCG to be calculated at + * the fourth position * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) + * / log_2(rank + 1) * ------------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0 | 7.0 - * 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 - * 3 | 3 | 7.0 | 2.0 | 3.5 - * 4 | n/a | n/a | n/a | n/a - * ----------------------------------------------------------------- - * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163 - * 6 | n/a | n/a | n/a | n/a + * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 + * | 1.8927892607143721 3 | 3 | 7.0 | 2.0 | 3.5 4 | n/a | n/a | n/a | n/a + * ----------------------------------------------------------------- 5 | 1 + * | 1.0 | 2.584962500721156 | 0.38685280723454163 6 | n/a | n/a | n/a | n/a * * dcg = 12.392789260714371 (sum of last column until position 4) */ public void testDCGAtFourMoreRatings() { - Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1, null}; + Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1, null }; List ratedDocs = new ArrayList<>(); for (int i = 0; i < 6; i++) { if (i < relevanceRatings.length) { if (relevanceRatings[i] != null) { - ratedDocs.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); + ratedDocs.add(new RatedDocument("index", "type", Integer.toString(i), + relevanceRatings[i])); } } } // only create four hits SearchHit[] hits = new SearchHit[4]; for (int i = 0; i < 4; i++) { - hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), + Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); - EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs); - assertEquals(12.392789260714371 , result.getQualityLevel(), 0.00001); + EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs); + assertEquals(12.392789260714371, result.getQualityLevel(), 0.00001); assertEquals(1, filterUnknownDocuments(result.getHitsAndRatings()).size()); /** - * Check with normalization: to get the maximal possible dcg, sort documents by relevance in descending order + * Check with normalization: to get the maximal possible dcg, sort + * documents by relevance in descending order * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) - * ------------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0  | 7.0 - * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 - * 3 | 2 | 3.0 | 2.0  | 1.5 - * 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339 - * ------------------------------------------------------------------------------------------- - * 5 | n.a | n.a | n.a.  | n.a. - * 6 | n.a | n.a | n.a  | n.a + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) + * - 1) / log_2(rank + 1) + * --------------------------------------------------------------------------------------- + * 1 | 3 | 7.0 | 1.0  | 7.0 2 | 3 | 7.0 | + * 1.5849625007211563 | 4.416508275000202 3 | 2 | 3.0 | 2.0  | 1.5 4 | 1 + * | 1.0 | 2.321928094887362   | 0.43067655807339 + * --------------------------------------------------------------------------------------- + * 5 | n.a | n.a | n.a.  | n.a. 6 | n.a | n.a | n.a  | n.a * * idcg = 13.347184833073591 (sum of last column) */ dcg.setNormalize(true); - assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), 0.00001); + assertEquals(12.392789260714371 / 13.347184833073591, + dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), 0.00001); } public void testParseFromXContent() throws IOException { - String xContent = " {\n" - + " \"unknown_doc_rating\": 2,\n" - + " \"normalize\": true\n" - + "}"; + String xContent = " {\n" + " \"unknown_doc_rating\": 2,\n" + " \"normalize\": true\n" + + "}"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) { DiscountedCumulativeGain dcgAt = DiscountedCumulativeGain.fromXContent(parser); assertEquals(2, dcgAt.getUnknownDocRating().intValue()); @@ -208,10 +214,12 @@ public class DiscountedCumulativeGainTests extends ESTestCase { return new DiscountedCumulativeGain(normalize, unknownDocRating); } + public void testXContentRoundtrip() throws IOException { DiscountedCumulativeGain testItem = createTestItem(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent( + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); itemParser.nextToken(); @@ -224,7 +232,8 @@ public class DiscountedCumulativeGainTests extends ESTestCase { public void testSerialization() throws IOException { DiscountedCumulativeGain original = createTestItem(); - DiscountedCumulativeGain deserialized = RankEvalTestHelper.copy(original, DiscountedCumulativeGain::new); + DiscountedCumulativeGain deserialized = RankEvalTestHelper.copy(original, + DiscountedCumulativeGain::new); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); @@ -236,7 +245,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase { RankEvalTestHelper.copy(testItem, DiscountedCumulativeGain::new)); } - private DiscountedCumulativeGain mutateTestItem(DiscountedCumulativeGain original) { + private static DiscountedCumulativeGain mutateTestItem(DiscountedCumulativeGain original) { boolean normalise = original.getNormalize(); int unknownDocRating = original.getUnknownDocRating(); DiscountedCumulativeGain gain = new DiscountedCumulativeGain(); @@ -244,8 +253,9 @@ public class DiscountedCumulativeGainTests extends ESTestCase { gain.setUnknownDocRating(unknownDocRating); List mutators = new ArrayList<>(); - mutators.add(() -> gain.setNormalize(! original.getNormalize())); - mutators.add(() -> gain.setUnknownDocRating(randomValueOtherThan(unknownDocRating, () -> randomIntBetween(0, 10)))); + mutators.add(() -> gain.setNormalize(!original.getNormalize())); + mutators.add(() -> gain.setUnknownDocRating( + randomValueOtherThan(unknownDocRating, () -> randomIntBetween(0, 10)))); randomFrom(mutators).run(); return gain; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java index 276dbffda73..5bd17730643 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -28,7 +28,8 @@ import java.util.List; public class EvalQueryQualityTests extends ESTestCase { - private static NamedWriteableRegistry namedWritableRegistry = new NamedWriteableRegistry(new RankEvalPlugin().getNamedWriteables()); + private static NamedWriteableRegistry namedWritableRegistry = new NamedWriteableRegistry( + new RankEvalPlugin().getNamedWriteables()); public static EvalQueryQuality randomEvalQueryQuality() { List unknownDocs = new ArrayList<>(); @@ -41,7 +42,8 @@ public class EvalQueryQualityTests extends ESTestCase { for (int i = 0; i < numberOfSearchHits; i++) { ratedHits.add(RatedSearchHitTests.randomRatedSearchHit()); } - EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAsciiOfLength(10), randomDoubleBetween(0.0, 1.0, true)); + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAsciiOfLength(10), + randomDoubleBetween(0.0, 1.0, true)); if (randomBoolean()) { // TODO randomize this evalQueryQuality.addMetricDetails(new Precision.Breakdown(1, 5)); @@ -52,7 +54,8 @@ public class EvalQueryQualityTests extends ESTestCase { public void testSerialization() throws IOException { EvalQueryQuality original = randomEvalQueryQuality(); - EvalQueryQuality deserialized = RankEvalTestHelper.copy(original, EvalQueryQuality::new, namedWritableRegistry); + EvalQueryQuality deserialized = RankEvalTestHelper.copy(original, EvalQueryQuality::new, + namedWritableRegistry); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java index 6e41960d823..be6a9b89c39 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java @@ -43,9 +43,11 @@ public class PrecisionTests extends ESTestCase { public void testPrecisionAtFiveCalculation() { List rated = new ArrayList<>(); rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated, "test", "testtype"), rated); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", + toSearchHits(rated, "test", "testtype"), rated); assertEquals(1, evaluated.getQualityLevel(), 0.00001); - assertEquals(1, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(1, + ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(1, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } @@ -56,15 +58,18 @@ public class PrecisionTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); - EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated, "test", "testtype"), rated); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", + toSearchHits(rated, "test", "testtype"), rated); assertEquals((double) 4 / 5, evaluated.getQualityLevel(), 0.00001); - assertEquals(4, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(4, + ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } /** - * test that the relevant rating threshold can be set to something larger than 1. - * e.g. we set it to 2 here and expect dics 0-2 to be not relevant, doc 3 and 4 to be relevant + * test that the relevant rating threshold can be set to something larger + * than 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant, + * doc 3 and 4 to be relevant */ public void testPrecisionAtFiveRelevanceThreshold() { List rated = new ArrayList<>(); @@ -75,9 +80,11 @@ public class PrecisionTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "4", 4)); Precision precisionAtN = new Precision(); precisionAtN.setRelevantRatingThreshhold(2); - EvalQueryQuality evaluated = precisionAtN.evaluate("id", toSearchHits(rated, "test", "testtype"), rated); + EvalQueryQuality evaluated = precisionAtN.evaluate("id", + toSearchHits(rated, "test", "testtype"), rated); assertEquals((double) 3 / 5, evaluated.getQualityLevel(), 0.00001); - assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(3, + ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } @@ -89,9 +96,11 @@ public class PrecisionTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "2", Rating.IRRELEVANT.ordinal())); // the following search hits contain only the last three documents - EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", + toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(2, + ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } @@ -102,9 +111,11 @@ public class PrecisionTests extends ESTestCase { rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "testtype", "2", Rating.IRRELEVANT.ordinal())); - EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", + toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(2, + ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } @@ -119,7 +130,8 @@ public class PrecisionTests extends ESTestCase { EvalQueryQuality evaluated = (new Precision()).evaluate("id", searchHits, rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(2, + ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); // also try with setting `ignore_unlabeled` @@ -127,19 +139,22 @@ public class PrecisionTests extends ESTestCase { prec.setIgnoreUnlabeled(true); evaluated = prec.evaluate("id", searchHits, rated); assertEquals((double) 2 / 2, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(2, + ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testNoRatedDocs() throws Exception { SearchHit[] hits = new SearchHit[5]; for (int i = 0; i < 5; i++) { - hits[i] = new SearchHit(i, i+"", new Text("type"), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + "", new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); } - EvalQueryQuality evaluated = (new Precision()).evaluate("id", hits, Collections.emptyList()); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", hits, + Collections.emptyList()); assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); - assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(0, + ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); // also try with setting `ignore_unlabeled` @@ -147,14 +162,13 @@ public class PrecisionTests extends ESTestCase { prec.setIgnoreUnlabeled(true); evaluated = prec.evaluate("id", hits, Collections.emptyList()); assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); - assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(0, + ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testParseFromXContent() throws IOException { - String xContent = " {\n" - + " \"relevant_rating_threshold\" : 2" - + "}"; + String xContent = " {\n" + " \"relevant_rating_threshold\" : 2" + "}"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) { Precision precicionAt = Precision.fromXContent(parser); assertEquals(2, precicionAt.getRelevantRatingThreshold()); @@ -187,7 +201,8 @@ public class PrecisionTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { Precision testItem = createTestItem(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent( + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); itemParser.nextToken(); @@ -220,8 +235,9 @@ public class PrecisionTests extends ESTestCase { precision.setRelevantRatingThreshhold(relevantThreshold); List mutators = new ArrayList<>(); - mutators.add(() -> precision.setIgnoreUnlabeled(! ignoreUnlabeled)); - mutators.add(() -> precision.setRelevantRatingThreshhold(randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10)))); + mutators.add(() -> precision.setIgnoreUnlabeled(!ignoreUnlabeled)); + mutators.add(() -> precision.setRelevantRatingThreshhold( + randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10)))); randomFrom(mutators).run(); return precision; } @@ -229,7 +245,7 @@ public class PrecisionTests extends ESTestCase { private static SearchHit[] toSearchHits(List rated, String index, String type) { SearchHit[] hits = new SearchHit[rated.size()]; for (int i = 0; i < rated.size(); i++) { - hits[i] = new SearchHit(i, i+"", new Text(type), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + "", new Text(type), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 50e9dcc2df9..49d5db32919 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -37,8 +37,7 @@ import java.util.Set; import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; - -public class RankEvalRequestIT extends ESIntegTestCase { +public class RankEvalRequestIT extends ESIntegTestCase { @Override protected Collection> transportClientPlugins() { return Arrays.asList(RankEvalPlugin.class); @@ -56,16 +55,11 @@ public class RankEvalRequestIT extends ESIntegTestCase { client().prepareIndex("test", "testtype").setId("1") .setSource("text", "berlin", "title", "Berlin, Germany").get(); - client().prepareIndex("test", "testtype").setId("2") - .setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("3") - .setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("4") - .setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("5") - .setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("6") - .setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("2").setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("3").setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("4").setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("5").setSource("text", "amsterdam").get(); + client().prepareIndex("test", "testtype").setId("6").setSource("text", "amsterdam").get(); refresh(); } @@ -76,29 +70,31 @@ public class RankEvalRequestIT extends ESIntegTestCase { List specifications = new ArrayList<>(); SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - RatedRequest amsterdamRequest = new RatedRequest( - "amsterdam_query", createRelevant("2", "3", "4", "5"), testQuery); + RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", + createRelevant("2", "3", "4", "5"), testQuery); amsterdamRequest.setIndices(indices); amsterdamRequest.setTypes(types); - amsterdamRequest.setSummaryFields(Arrays.asList(new String[]{ "text", "title" })); + amsterdamRequest.setSummaryFields(Arrays.asList(new String[] { "text", "title" })); specifications.add(amsterdamRequest); - RatedRequest berlinRequest = new RatedRequest( - "berlin_query", createRelevant("1"), testQuery); + RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("1"), + testQuery); berlinRequest.setIndices(indices); berlinRequest.setTypes(types); - berlinRequest.setSummaryFields(Arrays.asList(new String[]{ "text", "title" })); - + berlinRequest.setSummaryFields(Arrays.asList(new String[] { "text", "title" })); + specifications.add(berlinRequest); Precision metric = new Precision(); metric.setIgnoreUnlabeled(true); RankEvalSpec task = new RankEvalSpec(specifications, metric); - RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); + RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), + RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); - RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()) + .actionGet(); assertEquals(1.0, response.getQualityLevel(), Double.MIN_VALUE); Set> entrySet = response.getPartialResults().entrySet(); assertEquals(2, entrySet.size()); @@ -134,7 +130,8 @@ public class RankEvalRequestIT extends ESIntegTestCase { } /** - * test that running a bad query (e.g. one that will target a non existing field) will produce an error in the response + * test that running a bad query (e.g. one that will target a non existing + * field) will produce an error in the response */ public void testBadQuery() { List indices = Arrays.asList(new String[] { "test" }); @@ -143,7 +140,8 @@ public class RankEvalRequestIT extends ESIntegTestCase { List specifications = new ArrayList<>(); SearchSourceBuilder amsterdamQuery = new SearchSourceBuilder(); amsterdamQuery.query(new MatchAllQueryBuilder()); - RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", createRelevant("2", "3", "4", "5"), amsterdamQuery); + RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", + createRelevant("2", "3", "4", "5"), amsterdamQuery); amsterdamRequest.setIndices(indices); amsterdamRequest.setTypes(types); specifications.add(amsterdamRequest); @@ -151,20 +149,25 @@ public class RankEvalRequestIT extends ESIntegTestCase { SearchSourceBuilder brokenQuery = new SearchSourceBuilder(); RangeQueryBuilder brokenRangeQuery = new RangeQueryBuilder("text").timeZone("CET"); brokenQuery.query(brokenRangeQuery); - RatedRequest brokenRequest = new RatedRequest("broken_query", createRelevant("1"), brokenQuery); + RatedRequest brokenRequest = new RatedRequest("broken_query", createRelevant("1"), + brokenQuery); brokenRequest.setIndices(indices); brokenRequest.setTypes(types); specifications.add(brokenRequest); RankEvalSpec task = new RankEvalSpec(specifications, new Precision()); - RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); + RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), + RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); - RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); + RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()) + .actionGet(); assertEquals(1, response.getFailures().size()); - ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(response.getFailures().get("broken_query")); - assertEquals("[range] time_zone can not be applied to non date field [text]", rootCauses[0].getMessage()); + ElasticsearchException[] rootCauses = ElasticsearchException + .guessRootCauses(response.getFailures().get("broken_query")); + assertEquals("[range] time_zone can not be applied to non date field [text]", + rootCauses[0].getMessage()); } @@ -175,4 +178,4 @@ public class RankEvalRequestIT extends ESIntegTestCase { } return relevant; } - } +} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index fa1a512c54b..a89e73d149a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -45,13 +45,15 @@ public class RankEvalResponseTests extends ESTestCase { for (int d = 0; d < numberOfUnknownDocs; d++) { unknownDocs.add(DocumentKeyTests.createRandomRatedDocumentKey()); } - EvalQueryQuality evalQuality = new EvalQueryQuality(id, randomDoubleBetween(0.0, 1.0, true)); + EvalQueryQuality evalQuality = new EvalQueryQuality(id, + randomDoubleBetween(0.0, 1.0, true)); partials.put(id, evalQuality); } int numberOfErrors = randomIntBetween(0, 2); Map errors = new HashMap<>(numberOfRequests); for (int i = 0; i < numberOfErrors; i++) { - errors.put(randomAsciiOfLengthBetween(3, 10), new IllegalArgumentException(randomAsciiOfLength(10))); + errors.put(randomAsciiOfLengthBetween(3, 10), + new IllegalArgumentException(randomAsciiOfLength(10))); } return new RankEvalResponse(randomDouble(), partials, errors); } @@ -63,9 +65,12 @@ public class RankEvalResponseTests extends ESTestCase { try (StreamInput in = output.bytes().streamInput()) { RankEvalResponse deserializedResponse = new RankEvalResponse(); deserializedResponse.readFrom(in); - assertEquals(randomResponse.getQualityLevel(), deserializedResponse.getQualityLevel(), Double.MIN_VALUE); - assertEquals(randomResponse.getPartialResults(), deserializedResponse.getPartialResults()); - assertEquals(randomResponse.getFailures().keySet(), deserializedResponse.getFailures().keySet()); + assertEquals(randomResponse.getQualityLevel(), + deserializedResponse.getQualityLevel(), Double.MIN_VALUE); + assertEquals(randomResponse.getPartialResults(), + deserializedResponse.getPartialResults()); + assertEquals(randomResponse.getFailures().keySet(), + deserializedResponse.getFailures().keySet()); assertNotSame(randomResponse, deserializedResponse); assertEquals(-1, in.read()); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index b1ab9d9134a..4f1f900ee0d 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -68,7 +68,8 @@ public class RankEvalSpecTests extends ESTestCase { Collection templates = null; if (randomBoolean()) { - final Map params = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("key", "value"); + final Map params = randomBoolean() ? Collections.emptyMap() + : Collections.singletonMap("key", "value"); ScriptType scriptType = randomFrom(ScriptType.values()); String script; if (scriptType == ScriptType.INLINE) { @@ -83,17 +84,19 @@ public class RankEvalSpecTests extends ESTestCase { } templates = new HashSet<>(); - templates.add( - new ScriptWithId("templateId", new Script(scriptType, Script.DEFAULT_TEMPLATE_LANG, script, params))); + templates.add(new ScriptWithId("templateId", + new Script(scriptType, Script.DEFAULT_TEMPLATE_LANG, script, params))); Map templateParams = new HashMap<>(); templateParams.put("key", "value"); - RatedRequest ratedRequest = new RatedRequest( - "id", Arrays.asList(RatedDocumentTests.createRatedDocument()), templateParams, "templateId"); + RatedRequest ratedRequest = new RatedRequest("id", + Arrays.asList(RatedDocumentTests.createRatedDocument()), templateParams, + "templateId"); ratedRequests = Arrays.asList(ratedRequest); } else { - RatedRequest ratedRequest = new RatedRequest( - "id", Arrays.asList(RatedDocumentTests.createRatedDocument()), new SearchSourceBuilder()); + RatedRequest ratedRequest = new RatedRequest("id", + Arrays.asList(RatedDocumentTests.createRatedDocument()), + new SearchSourceBuilder()); ratedRequests = Arrays.asList(ratedRequest); } RankEvalSpec spec = new RankEvalSpec(ratedRequests, metric, templates); @@ -104,7 +107,8 @@ public class RankEvalSpecTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { RankEvalSpec testItem = createTestItem(); - XContentBuilder shuffled = shuffleXContent(testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent( + testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); try (XContentParser parser = createParser(JsonXContent.jsonXContent, shuffled.bytes())) { RankEvalSpec parsedItem = RankEvalSpec.parse(parser); @@ -122,14 +126,17 @@ public class RankEvalSpecTests extends ESTestCase { RankEvalSpec original = createTestItem(); List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry( - RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, + MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, + Precision.NAME, Precision::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, + DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, + ReciprocalRank.NAME, ReciprocalRank::new)); - - RankEvalSpec deserialized = RankEvalTestHelper.copy(original, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables)); + RankEvalSpec deserialized = RankEvalTestHelper.copy(original, RankEvalSpec::new, + new NamedWriteableRegistry(namedWriteables)); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); @@ -139,15 +146,20 @@ public class RankEvalSpecTests extends ESTestCase { RankEvalSpec testItem = createTestItem(); List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry( - RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, + MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, + Precision.NAME, Precision::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, + DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, + ReciprocalRank.NAME, ReciprocalRank::new)); - RankEvalSpec mutant = RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables)); + RankEvalSpec mutant = RankEvalTestHelper.copy(testItem, RankEvalSpec::new, + new NamedWriteableRegistry(namedWriteables)); RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(mutant), - RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables))); + RankEvalTestHelper.copy(testItem, RankEvalSpec::new, + new NamedWriteableRegistry(namedWriteables))); } private static RankEvalSpec mutateTestItem(RankEvalSpec mutant) { @@ -157,28 +169,32 @@ public class RankEvalSpecTests extends ESTestCase { int mutate = randomIntBetween(0, 2); switch (mutate) { - case 0: - RatedRequest request = RatedRequestsTests.createTestItem(new ArrayList<>(), new ArrayList<>(), true); - ratedRequests.add(request); - break; - case 1: - if (metric instanceof Precision) { - metric = new DiscountedCumulativeGain(); - } else { - metric = new Precision(); - } - break; - case 2: - if (templates.size() > 0) { - String mutatedTemplate = randomAsciiOfLength(10); - templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, new HashMap<>())); - } else { - String mutatedTemplate = randomValueOtherThanMany(templates::containsValue, () -> randomAsciiOfLength(10)); - templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, new HashMap<>())); - } - break; - default: - throw new IllegalStateException("Requested to modify more than available parameters."); + case 0: + RatedRequest request = RatedRequestsTests.createTestItem(new ArrayList<>(), + new ArrayList<>(), true); + ratedRequests.add(request); + break; + case 1: + if (metric instanceof Precision) { + metric = new DiscountedCumulativeGain(); + } else { + metric = new Precision(); + } + break; + case 2: + if (templates.size() > 0) { + String mutatedTemplate = randomAsciiOfLength(10); + templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, + new HashMap<>())); + } else { + String mutatedTemplate = randomValueOtherThanMany(templates::containsValue, + () -> randomAsciiOfLength(10)); + templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, + new HashMap<>())); + } + break; + default: + throw new IllegalStateException("Requested to modify more than available parameters."); } List scripts = new ArrayList<>(); @@ -192,24 +208,28 @@ public class RankEvalSpecTests extends ESTestCase { public void testMissingRatedRequestsFailsParsing() { RankedListQualityMetric metric = new Precision(); - expectThrows(IllegalStateException.class, () -> new RankEvalSpec(new ArrayList<>(), metric)); + expectThrows(IllegalStateException.class, + () -> new RankEvalSpec(new ArrayList<>(), metric)); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(null, metric)); } public void testMissingMetricFailsParsing() { List strings = Arrays.asList("value"); - List ratedRequests = randomList(() -> RatedRequestsTests.createTestItem(strings, strings, randomBoolean())); + List ratedRequests = randomList( + () -> RatedRequestsTests.createTestItem(strings, strings, randomBoolean())); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, null)); } public void testMissingTemplateAndSearchRequestFailsParsing() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + List ratedDocs = Arrays + .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); Map params = new HashMap<>(); params.put("key", "value"); RatedRequest request = new RatedRequest("id", ratedDocs, params, "templateId"); List ratedRequests = Arrays.asList(request); - expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, new Precision())); + expectThrows(IllegalStateException.class, + () -> new RankEvalSpec(ratedRequests, new Precision())); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java index be97ad8ff06..15ae5d03537 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java @@ -42,40 +42,50 @@ public class RankEvalTestHelper { assertFalse("testItem is equal to null", testItem.equals(null)); assertFalse("testItem is equal to incompatible type", testItem.equals("")); assertTrue("testItem is not equal to self", testItem.equals(testItem)); - assertThat("same testItem's hashcode returns different values if called multiple times", testItem.hashCode(), - equalTo(testItem.hashCode())); + assertThat("same testItem's hashcode returns different values if called multiple times", + testItem.hashCode(), equalTo(testItem.hashCode())); assertThat("different testItem should not be equal", mutation, not(equalTo(testItem))); assertNotSame("testItem copy is not same as original", testItem, secondCopy); assertTrue("testItem is not equal to its copy", testItem.equals(secondCopy)); assertTrue("equals is not symmetric", secondCopy.equals(testItem)); - assertThat("testItem copy's hashcode is different from original hashcode", secondCopy.hashCode(), - equalTo(testItem.hashCode())); + assertThat("testItem copy's hashcode is different from original hashcode", + secondCopy.hashCode(), equalTo(testItem.hashCode())); } /** * Make a deep copy of an object by running it through a BytesStreamOutput - * @param original the original object - * @param reader a function able to create a new copy of this type + * + * @param original + * the original object + * @param reader + * a function able to create a new copy of this type * @return a new copy of the original object */ - public static T copy(T original, Writeable.Reader reader) throws IOException { + public static T copy(T original, Writeable.Reader reader) + throws IOException { return copy(original, reader, new NamedWriteableRegistry(Collections.emptyList())); } /** * Make a deep copy of an object by running it through a BytesStreamOutput - * @param original the original object - * @param reader a function able to create a new copy of this type - * @param namedWriteableRegistry must be non-empty if the object itself or nested object implement {@link NamedWriteable} + * + * @param original + * the original object + * @param reader + * a function able to create a new copy of this type + * @param namedWriteableRegistry + * must be non-empty if the object itself or nested object + * implement {@link NamedWriteable} * @return a new copy of the original object */ - public static T copy(T original, Writeable.Reader reader, NamedWriteableRegistry namedWriteableRegistry) - throws IOException { + public static T copy(T original, Writeable.Reader reader, + NamedWriteableRegistry namedWriteableRegistry) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), + namedWriteableRegistry)) { return reader.read(in); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index b100a21b38b..ba8b4a25176 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -42,7 +42,8 @@ public class RatedDocumentTests extends ESTestCase { public void testXContentParsing() throws IOException { RatedDocument testItem = createRatedDocument(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent( + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { RatedDocument parsedItem = RatedDocument.fromXContent(itemParser); assertNotSame(testItem, parsedItem); @@ -66,11 +67,14 @@ public class RatedDocumentTests extends ESTestCase { } public void testInvalidParsing() throws IOException { - expectThrows(IllegalArgumentException.class, () -> new RatedDocument(null, "abc", "abc", 10)); + expectThrows(IllegalArgumentException.class, + () -> new RatedDocument(null, "abc", "abc", 10)); expectThrows(IllegalArgumentException.class, () -> new RatedDocument("", "abc", "abc", 10)); - expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", null, "abc", 10)); + expectThrows(IllegalArgumentException.class, + () -> new RatedDocument("abc", null, "abc", 10)); expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "", "abc", 10)); - expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "abc", null, 10)); + expectThrows(IllegalArgumentException.class, + () -> new RatedDocument("abc", "abc", null, 10)); expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "abc", "", 10)); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 99035504785..366d0ba5151 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -53,13 +53,13 @@ public class RatedRequestsTests extends ESTestCase { private static NamedXContentRegistry xContentRegistry; /** - * setup for the whole base test class - */ + * setup for the whole base test class + */ @BeforeClass public static void init() { xContentRegistry = new NamedXContentRegistry(Stream.of( - new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents().stream() - ).flatMap(Function.identity()).collect(toList())); + new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents().stream()) + .flatMap(Function.identity()).collect(toList())); } @AfterClass @@ -72,7 +72,8 @@ public class RatedRequestsTests extends ESTestCase { return xContentRegistry; } - public static RatedRequest createTestItem(List indices, List types, boolean forceRequest) { + public static RatedRequest createTestItem(List indices, List types, + boolean forceRequest) { String requestId = randomAsciiOfLength(50); List ratedDocs = new ArrayList<>(); @@ -130,7 +131,8 @@ public class RatedRequestsTests extends ESTestCase { RatedRequest testItem = createTestItem(indices, types, randomBoolean()); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent( + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); @@ -162,9 +164,11 @@ public class RatedRequestsTests extends ESTestCase { RatedRequest original = createTestItem(indices, types, randomBoolean()); List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, + MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - RatedRequest deserialized = RankEvalTestHelper.copy(original, RatedRequest::new, new NamedWriteableRegistry(namedWriteables)); + RatedRequest deserialized = RankEvalTestHelper.copy(original, RatedRequest::new, + new NamedWriteableRegistry(namedWriteables)); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); @@ -186,10 +190,12 @@ public class RatedRequestsTests extends ESTestCase { RatedRequest testItem = createTestItem(indices, types, randomBoolean()); List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, + MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - RankEvalTestHelper.copy(testItem, RatedRequest::new, new NamedWriteableRegistry(namedWriteables))); + RankEvalTestHelper.copy(testItem, RatedRequest::new, + new NamedWriteableRegistry(namedWriteables))); } private RatedRequest mutateTestItem(RatedRequest original) { @@ -204,44 +210,48 @@ public class RatedRequestsTests extends ESTestCase { int mutate = randomIntBetween(0, 5); switch (mutate) { - case 0: - id = randomValueOtherThan(id, () -> randomAsciiOfLength(10)); - break; - case 1: - if (testRequest != null) { - int size = randomValueOtherThan(testRequest.size(), () -> randomInt()); - testRequest = new SearchSourceBuilder(); - testRequest.size(size); - testRequest.query(new MatchAllQueryBuilder()); + case 0: + id = randomValueOtherThan(id, () -> randomAsciiOfLength(10)); + break; + case 1: + if (testRequest != null) { + int size = randomValueOtherThan(testRequest.size(), () -> randomInt()); + testRequest = new SearchSourceBuilder(); + testRequest.size(size); + testRequest.query(new MatchAllQueryBuilder()); + } else { + if (randomBoolean()) { + Map mutated = new HashMap<>(); + mutated.putAll(params); + mutated.put("one_more_key", "one_more_value"); + params = mutated; } else { - if (randomBoolean()) { - Map mutated = new HashMap<>(); - mutated.putAll(params); - mutated.put("one_more_key", "one_more_value"); - params = mutated; - } else { - templateId = randomValueOtherThan(templateId, () -> randomAsciiOfLength(5)); - } + templateId = randomValueOtherThan(templateId, () -> randomAsciiOfLength(5)); } - break; - case 2: - ratedDocs = Arrays.asList( - randomValueOtherThanMany(ratedDocs::contains, () -> RatedDocumentTests.createRatedDocument())); - break; - case 3: - indices = Arrays.asList(randomValueOtherThanMany(indices::contains, () -> randomAsciiOfLength(10))); - break; - case 4: - types = Arrays.asList(randomValueOtherThanMany(types::contains, () -> randomAsciiOfLength(10))); - break; - case 5: - summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains, () -> randomAsciiOfLength(10))); - break; - default: - throw new IllegalStateException("Requested to modify more than available parameters."); + } + break; + case 2: + ratedDocs = Arrays.asList(randomValueOtherThanMany(ratedDocs::contains, + () -> RatedDocumentTests.createRatedDocument())); + break; + case 3: + indices = Arrays.asList( + randomValueOtherThanMany(indices::contains, () -> randomAsciiOfLength(10))); + break; + case 4: + types = Arrays.asList( + randomValueOtherThanMany(types::contains, () -> randomAsciiOfLength(10))); + break; + case 5: + summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains, + () -> randomAsciiOfLength(10))); + break; + default: + throw new IllegalStateException("Requested to modify more than available parameters."); } - RatedRequest ratedRequest = new RatedRequest(id, ratedDocs, testRequest, params, templateId); + RatedRequest ratedRequest = new RatedRequest(id, ratedDocs, testRequest, params, + templateId); ratedRequest.setIndices(indices); ratedRequest.setTypes(types); ratedRequest.setSummaryFields(summaryFields); @@ -250,41 +260,46 @@ public class RatedRequestsTests extends ESTestCase { } public void testDuplicateRatedDocThrowsException() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1), + List ratedDocs = Arrays.asList( + new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1), new RatedDocument(new DocumentKey("index1", "type1", "id1"), 5)); // search request set, no summary fields - IllegalArgumentException ex = expectThrows( - IllegalArgumentException.class, + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder())); assertEquals( - "Found duplicate rated document key [{ \"_index\" : \"index1\", \"_type\" : \"type1\", \"_id\" : \"id1\"}]", + "Found duplicate rated document key [{ \"_index\" : \"index1\", " + + "\"_type\" : \"type1\", \"_id\" : \"id1\"}]", ex.getMessage()); // templated path, no summary fields Map params = new HashMap<>(); params.put("key", "value"); - ex = expectThrows( - IllegalArgumentException.class, + ex = expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, params, "templateId")); assertEquals( - "Found duplicate rated document key [{ \"_index\" : \"index1\", \"_type\" : \"type1\", \"_id\" : \"id1\"}]", + "Found duplicate rated document key [{ \"_index\" : \"index1\", " + + "\"_type\" : \"type1\", \"_id\" : \"id1\"}]", ex.getMessage()); } public void testNullSummaryFieldsTreatment() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + List ratedDocs = Arrays + .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder()); expectThrows(IllegalArgumentException.class, () -> request.setSummaryFields(null)); } public void testNullParamsTreatment() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); - RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, null); + List ratedDocs = Arrays + .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, + null); assertNotNull(request.getParams()); } public void testSettingParamsAndRequestThrows() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + List ratedDocs = Arrays + .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); Map params = new HashMap<>(); params.put("key", "value"); expectThrows(IllegalArgumentException.class, @@ -292,13 +307,17 @@ public class RatedRequestsTests extends ESTestCase { } public void testSettingNeitherParamsNorRequestThrows() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); - expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null)); - expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, new HashMap<>(), "templateId")); + List ratedDocs = Arrays + .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + expectThrows(IllegalArgumentException.class, + () -> new RatedRequest("id", ratedDocs, null, null)); + expectThrows(IllegalArgumentException.class, + () -> new RatedRequest("id", ratedDocs, null, new HashMap<>(), "templateId")); } public void testSettingParamsWithoutTemplateIdThrows() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + List ratedDocs = Arrays + .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); Map params = new HashMap<>(); params.put("key", "value"); expectThrows(IllegalArgumentException.class, @@ -306,38 +325,43 @@ public class RatedRequestsTests extends ESTestCase { } public void testSettingTemplateIdAndRequestThrows() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); - expectThrows(IllegalArgumentException.class, - () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, "templateId")); + List ratedDocs = Arrays + .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, + new SearchSourceBuilder(), null, "templateId")); } public void testSettingTemplateIdNoParamsThrows() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + List ratedDocs = Arrays + .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null, "templateId")); } + /** + * test that modifying the order of index/type/docId to make sure it doesn't + * matter for parsing xContent + */ public void testParseFromXContent() throws IOException { - // we modify the order of index/type/docId to make sure it doesn't matter for parsing xContent - String querySpecString = " {\n" - + " \"id\": \"my_qa_query\",\n" - + " \"request\": {\n" - + " \"query\": {\n" - + " \"bool\": {\n" - + " \"must\": [\n" - + " {\"match\": {\"beverage\": \"coffee\"}},\n" - + " {\"term\": {\"browser\": {\"value\": \"safari\"}}},\n" - + " {\"term\": {\"time_of_day\": {\"value\": \"morning\",\"boost\": 2}}},\n" - + " {\"term\": {\"ip_location\": {\"value\": \"ams\",\"boost\": 10}}}]}\n" - + " },\n" - + " \"size\": 10\n" - + " },\n" - + " \"summary_fields\" : [\"title\"],\n" - + " \"ratings\": [ " - + " {\"_index\": \"test\", \"_type\": \"testtype\", \"_id\": \"1\", \"rating\" : 1 }, " - + " {\"_type\": \"testtype\", \"_index\": \"test\", \"_id\": \"2\", \"rating\" : 0 }, " - + " {\"_id\": \"3\", \"_index\": \"test\", \"_type\": \"testtype\", \"rating\" : 1 }]\n" - + "}"; + String querySpecString = " {\n" + " \"id\": \"my_qa_query\",\n" + " \"request\": {\n" + + " \"query\": {\n" + " \"bool\": {\n" + + " \"must\": [\n" + + " {\"match\": {\"beverage\": \"coffee\"}},\n" + + " {\"term\": {\"browser\": {\"value\": \"safari\"}}},\n" + + " {\"term\": {\"time_of_day\": " + + " {\"value\": \"morning\",\"boost\": 2}}},\n" + + " {\"term\": {\"ip_location\": " + + " {\"value\": \"ams\",\"boost\": 10}}}]}\n" + + " },\n" + " \"size\": 10\n" + " },\n" + + " \"summary_fields\" : [\"title\"],\n" + + " \"ratings\": [\n" + + " {\"_index\": \"test\", \"_type\": \"testtype\", " + + " \"_id\": \"1\", \"rating\" : 1 }, " + + " {\"_type\": \"testtype\", \"_index\": \"test\", " + + " \"_id\": \"2\", \"rating\" : 0 }, " + + " {\"_id\": \"3\", \"_index\": \"test\", " + + " \"_type\": \"testtype\", \"rating\" : 1 }]\n" + + "}"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, querySpecString)) { RatedRequest specification = RatedRequest.fromXContent(parser); assertEquals("my_qa_query", specification.getId()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java index 4e5f5e2fc88..91a9509190e 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java @@ -30,9 +30,10 @@ import java.util.Optional; public class RatedSearchHitTests extends ESTestCase { public static RatedSearchHit randomRatedSearchHit() { - Optional rating = randomBoolean() ? Optional.empty() : Optional.of(randomIntBetween(0, 5)); - SearchHit searchHit = new SearchHit(randomIntBetween(0, 10), randomAsciiOfLength(10), new Text(randomAsciiOfLength(10)), - Collections.emptyMap()); + Optional rating = randomBoolean() ? Optional.empty() + : Optional.of(randomIntBetween(0, 5)); + SearchHit searchHit = new SearchHit(randomIntBetween(0, 10), randomAsciiOfLength(10), + new Text(randomAsciiOfLength(10)), Collections.emptyMap()); RatedSearchHit ratedSearchHit = new RatedSearchHit(searchHit, rating); return ratedSearchHit; } @@ -45,8 +46,8 @@ public class RatedSearchHitTests extends ESTestCase { rating = rating.isPresent() ? Optional.of(rating.get() + 1) : Optional.of(randomInt(5)); break; case 1: - hit = new SearchHit(hit.docId(), hit.getId() + randomAsciiOfLength(10), new Text(hit.getType()), - Collections.emptyMap()); + hit = new SearchHit(hit.docId(), hit.getId() + randomAsciiOfLength(10), + new Text(hit.getType()), Collections.emptyMap()); break; default: throw new IllegalStateException("The test should only allow two parameters mutated"); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 1f666f4c499..60eb6325a9a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -50,20 +50,25 @@ public class ReciprocalRankTests extends ESTestCase { int relevantAt = randomIntBetween(0, searchHits); for (int i = 0; i <= searchHits; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), + Rating.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), + Rating.IRRELEVANT.ordinal())); } } int rankAtFirstRelevant = relevantAt + 1; EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE); - assertEquals(rankAtFirstRelevant, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); + assertEquals(rankAtFirstRelevant, + ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); - // check that if we have fewer search hits than relevant doc position, we don't find any result and get 0.0 quality level + // check that if we have fewer search hits than relevant doc position, + // we don't find any result and get 0.0 quality level reciprocalRank = new ReciprocalRank(); - evaluation = reciprocalRank.evaluate("id", Arrays.copyOfRange(hits, 0, relevantAt), ratedDocs); + evaluation = reciprocalRank.evaluate("id", Arrays.copyOfRange(hits, 0, relevantAt), + ratedDocs); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); } @@ -75,21 +80,24 @@ public class ReciprocalRankTests extends ESTestCase { int relevantAt = randomIntBetween(0, 9); for (int i = 0; i <= 20; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), + Rating.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), + Rating.IRRELEVANT.ordinal())); } } EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE); - assertEquals(relevantAt + 1, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); + assertEquals(relevantAt + 1, + ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } /** - * test that the relevant rating threshold can be set to something larger than 1. - * e.g. we set it to 2 here and expect dics 0-2 to be not relevant, so first relevant doc has - * third ranking position, so RR should be 1/3 + * test that the relevant rating threshold can be set to something larger + * than 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant, + * so first relevant doc has third ranking position, so RR should be 1/3 */ public void testPrecisionAtFiveRelevanceThreshold() { List rated = new ArrayList<>(); @@ -104,7 +112,8 @@ public class ReciprocalRankTests extends ESTestCase { reciprocalRank.setRelevantRatingThreshhold(2); EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, rated); assertEquals((double) 1 / 3, evaluation.getQualityLevel(), 0.00001); - assertEquals(3, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); + assertEquals(3, + ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } public void testCombine() { @@ -127,7 +136,8 @@ public class ReciprocalRankTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { ReciprocalRank testItem = createTestItem(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent( + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); itemParser.nextToken(); @@ -139,13 +149,13 @@ public class ReciprocalRankTests extends ESTestCase { } /** - * Create SearchHits for testing, starting from dociId 'from' up to docId 'to'. - * The search hits index and type also need to be provided + * Create SearchHits for testing, starting from dociId 'from' up to docId + * 'to'. The search hits index and type also need to be provided */ private static SearchHit[] createSearchHits(int from, int to, String index, String type) { SearchHit[] hits = new SearchHit[to + 1 - from]; for (int i = from; i <= to; i++) { - hits[i] = new SearchHit(i, i+"", new Text(type), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + "", new Text(type), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0)); } return hits; @@ -175,7 +185,8 @@ public class ReciprocalRankTests extends ESTestCase { private ReciprocalRank mutateTestItem(ReciprocalRank testItem) { int relevantThreshold = testItem.getRelevantRatingThreshold(); ReciprocalRank rank = new ReciprocalRank(); - rank.setRelevantRatingThreshhold(randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10))); + rank.setRelevantRatingThreshhold( + randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10))); return rank; } From 6cfbef73a00dcf741eae4790cef0230d6f49cc06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 4 Apr 2017 18:31:00 +0200 Subject: [PATCH 095/297] Follow renaming of randomAsciiOfLength() to randomAlphaOfLength() --- .../index/rankeval/DocumentKeyTests.java | 6 ++-- .../index/rankeval/EvalQueryQualityTests.java | 2 +- .../index/rankeval/RankEvalResponseTests.java | 6 ++-- .../index/rankeval/RankEvalSpecTests.java | 8 ++--- .../index/rankeval/RatedDocumentTests.java | 12 ++++---- .../index/rankeval/RatedRequestsTests.java | 30 +++++++++---------- .../index/rankeval/RatedSearchHitTests.java | 6 ++-- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java index 992f3f49fb1..1b3efa996c8 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java @@ -27,9 +27,9 @@ import java.io.IOException; public class DocumentKeyTests extends ESTestCase { static DocumentKey createRandomRatedDocumentKey() { - String index = randomAsciiOfLengthBetween(1, 10); - String type = randomAsciiOfLengthBetween(1, 10); - String docId = randomAsciiOfLengthBetween(1, 10); + String index = randomAlphaOfLengthBetween(1, 10); + String type = randomAlphaOfLengthBetween(1, 10); + String docId = randomAlphaOfLengthBetween(1, 10); return new DocumentKey(index, type, docId); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java index 5bd17730643..b938dd91d12 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -42,7 +42,7 @@ public class EvalQueryQualityTests extends ESTestCase { for (int i = 0; i < numberOfSearchHits; i++) { ratedHits.add(RatedSearchHitTests.randomRatedSearchHit()); } - EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAsciiOfLength(10), + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAlphaOfLength(10), randomDoubleBetween(0.0, 1.0, true)); if (randomBoolean()) { // TODO randomize this diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index a89e73d149a..497f17de1b9 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -39,7 +39,7 @@ public class RankEvalResponseTests extends ESTestCase { int numberOfRequests = randomIntBetween(0, 5); Map partials = new HashMap<>(numberOfRequests); for (int i = 0; i < numberOfRequests; i++) { - String id = randomAsciiOfLengthBetween(3, 10); + String id = randomAlphaOfLengthBetween(3, 10); int numberOfUnknownDocs = randomIntBetween(0, 5); List unknownDocs = new ArrayList<>(numberOfUnknownDocs); for (int d = 0; d < numberOfUnknownDocs; d++) { @@ -52,8 +52,8 @@ public class RankEvalResponseTests extends ESTestCase { int numberOfErrors = randomIntBetween(0, 2); Map errors = new HashMap<>(numberOfRequests); for (int i = 0; i < numberOfErrors; i++) { - errors.put(randomAsciiOfLengthBetween(3, 10), - new IllegalArgumentException(randomAsciiOfLength(10))); + errors.put(randomAlphaOfLengthBetween(3, 10), + new IllegalArgumentException(randomAlphaOfLength(10))); } return new RankEvalResponse(randomDouble(), partials, errors); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 4f1f900ee0d..f8a879a74fb 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -75,12 +75,12 @@ public class RankEvalSpecTests extends ESTestCase { if (scriptType == ScriptType.INLINE) { try (XContentBuilder builder = XContentFactory.jsonBuilder()) { builder.startObject(); - builder.field("field", randomAsciiOfLengthBetween(1, 5)); + builder.field("field", randomAlphaOfLengthBetween(1, 5)); builder.endObject(); script = builder.string(); } } else { - script = randomAsciiOfLengthBetween(1, 5); + script = randomAlphaOfLengthBetween(1, 5); } templates = new HashSet<>(); @@ -183,12 +183,12 @@ public class RankEvalSpecTests extends ESTestCase { break; case 2: if (templates.size() > 0) { - String mutatedTemplate = randomAsciiOfLength(10); + String mutatedTemplate = randomAlphaOfLength(10); templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, new HashMap<>())); } else { String mutatedTemplate = randomValueOtherThanMany(templates::containsValue, - () -> randomAsciiOfLength(10)); + () -> randomAlphaOfLength(10)); templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, new HashMap<>())); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index ba8b4a25176..3afbcef2a02 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -31,9 +31,9 @@ import java.io.IOException; public class RatedDocumentTests extends ESTestCase { public static RatedDocument createRatedDocument() { - String index = randomAsciiOfLength(10); - String type = randomAsciiOfLength(10); - String docId = randomAsciiOfLength(10); + String index = randomAlphaOfLength(10); + String type = randomAlphaOfLength(10); + String docId = randomAlphaOfLength(10); int rating = randomInt(); return new RatedDocument(index, type, docId, rating); @@ -89,13 +89,13 @@ public class RatedDocumentTests extends ESTestCase { rating = randomValueOtherThan(rating, () -> randomInt()); break; case 1: - index = randomValueOtherThan(index, () -> randomAsciiOfLength(10)); + index = randomValueOtherThan(index, () -> randomAlphaOfLength(10)); break; case 2: - type = randomValueOtherThan(type, () -> randomAsciiOfLength(10)); + type = randomValueOtherThan(type, () -> randomAlphaOfLength(10)); break; case 3: - docId = randomValueOtherThan(docId, () -> randomAsciiOfLength(10)); + docId = randomValueOtherThan(docId, () -> randomAlphaOfLength(10)); break; default: throw new IllegalStateException("The test should only allow two parameters mutated"); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 366d0ba5151..5d5ae92d4be 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -74,7 +74,7 @@ public class RatedRequestsTests extends ESTestCase { public static RatedRequest createTestItem(List indices, List types, boolean forceRequest) { - String requestId = randomAsciiOfLength(50); + String requestId = randomAlphaOfLength(50); List ratedDocs = new ArrayList<>(); int size = randomIntBetween(0, 2); @@ -91,14 +91,14 @@ public class RatedRequestsTests extends ESTestCase { } else { int randomSize = randomIntBetween(1, 10); for (int i = 0; i < randomSize; i++) { - params.put(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); + params.put(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)); } } List summaryFields = new ArrayList<>(); int numSummaryFields = randomIntBetween(0, 5); for (int i = 0; i < numSummaryFields; i++) { - summaryFields.add(randomAsciiOfLength(5)); + summaryFields.add(randomAlphaOfLength(5)); } RatedRequest ratedRequest = null; @@ -108,7 +108,7 @@ public class RatedRequestsTests extends ESTestCase { ratedRequest.setTypes(types); ratedRequest.setSummaryFields(summaryFields); } else { - ratedRequest = new RatedRequest(requestId, ratedDocs, params, randomAsciiOfLength(5)); + ratedRequest = new RatedRequest(requestId, ratedDocs, params, randomAlphaOfLength(5)); ratedRequest.setIndices(indices); ratedRequest.setTypes(types); ratedRequest.setSummaryFields(summaryFields); @@ -120,13 +120,13 @@ public class RatedRequestsTests extends ESTestCase { List indices = new ArrayList<>(); int size = randomIntBetween(0, 20); for (int i = 0; i < size; i++) { - indices.add(randomAsciiOfLengthBetween(0, 50)); + indices.add(randomAlphaOfLengthBetween(0, 50)); } List types = new ArrayList<>(); size = randomIntBetween(0, 20); for (int i = 0; i < size; i++) { - types.add(randomAsciiOfLengthBetween(0, 50)); + types.add(randomAlphaOfLengthBetween(0, 50)); } RatedRequest testItem = createTestItem(indices, types, randomBoolean()); @@ -152,13 +152,13 @@ public class RatedRequestsTests extends ESTestCase { List indices = new ArrayList<>(); int size = randomIntBetween(0, 20); for (int i = 0; i < size; i++) { - indices.add(randomAsciiOfLengthBetween(0, 50)); + indices.add(randomAlphaOfLengthBetween(0, 50)); } List types = new ArrayList<>(); size = randomIntBetween(0, 20); for (int i = 0; i < size; i++) { - types.add(randomAsciiOfLengthBetween(0, 50)); + types.add(randomAlphaOfLengthBetween(0, 50)); } RatedRequest original = createTestItem(indices, types, randomBoolean()); @@ -178,13 +178,13 @@ public class RatedRequestsTests extends ESTestCase { List indices = new ArrayList<>(); int size = randomIntBetween(0, 20); for (int i = 0; i < size; i++) { - indices.add(randomAsciiOfLengthBetween(0, 50)); + indices.add(randomAlphaOfLengthBetween(0, 50)); } List types = new ArrayList<>(); size = randomIntBetween(0, 20); for (int i = 0; i < size; i++) { - types.add(randomAsciiOfLengthBetween(0, 50)); + types.add(randomAlphaOfLengthBetween(0, 50)); } RatedRequest testItem = createTestItem(indices, types, randomBoolean()); @@ -211,7 +211,7 @@ public class RatedRequestsTests extends ESTestCase { int mutate = randomIntBetween(0, 5); switch (mutate) { case 0: - id = randomValueOtherThan(id, () -> randomAsciiOfLength(10)); + id = randomValueOtherThan(id, () -> randomAlphaOfLength(10)); break; case 1: if (testRequest != null) { @@ -226,7 +226,7 @@ public class RatedRequestsTests extends ESTestCase { mutated.put("one_more_key", "one_more_value"); params = mutated; } else { - templateId = randomValueOtherThan(templateId, () -> randomAsciiOfLength(5)); + templateId = randomValueOtherThan(templateId, () -> randomAlphaOfLength(5)); } } break; @@ -236,15 +236,15 @@ public class RatedRequestsTests extends ESTestCase { break; case 3: indices = Arrays.asList( - randomValueOtherThanMany(indices::contains, () -> randomAsciiOfLength(10))); + randomValueOtherThanMany(indices::contains, () -> randomAlphaOfLength(10))); break; case 4: types = Arrays.asList( - randomValueOtherThanMany(types::contains, () -> randomAsciiOfLength(10))); + randomValueOtherThanMany(types::contains, () -> randomAlphaOfLength(10))); break; case 5: summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains, - () -> randomAsciiOfLength(10))); + () -> randomAlphaOfLength(10))); break; default: throw new IllegalStateException("Requested to modify more than available parameters."); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java index 91a9509190e..3899a2e2029 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java @@ -32,8 +32,8 @@ public class RatedSearchHitTests extends ESTestCase { public static RatedSearchHit randomRatedSearchHit() { Optional rating = randomBoolean() ? Optional.empty() : Optional.of(randomIntBetween(0, 5)); - SearchHit searchHit = new SearchHit(randomIntBetween(0, 10), randomAsciiOfLength(10), - new Text(randomAsciiOfLength(10)), Collections.emptyMap()); + SearchHit searchHit = new SearchHit(randomIntBetween(0, 10), randomAlphaOfLength(10), + new Text(randomAlphaOfLength(10)), Collections.emptyMap()); RatedSearchHit ratedSearchHit = new RatedSearchHit(searchHit, rating); return ratedSearchHit; } @@ -46,7 +46,7 @@ public class RatedSearchHitTests extends ESTestCase { rating = rating.isPresent() ? Optional.of(rating.get() + 1) : Optional.of(randomInt(5)); break; case 1: - hit = new SearchHit(hit.docId(), hit.getId() + randomAsciiOfLength(10), + hit = new SearchHit(hit.docId(), hit.getId() + randomAlphaOfLength(10), new Text(hit.getType()), Collections.emptyMap()); break; default: From d1703deceeded694d180b379efac888f796cbcde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Sat, 22 Apr 2017 22:06:06 +0200 Subject: [PATCH 096/297] Adapting to changes in master --- .../java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java index 90c07bb244d..466cb92661d 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java @@ -25,15 +25,13 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import java.io.IOException; - public class RankEvalYamlIT extends ESClientYamlSuiteTestCase { public RankEvalYamlIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } @ParametersFactory - public static Iterable parameters() throws IOException { + public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } } From 10d308578eabda0a3cddc0f5d8621973ab5cb0d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 18 May 2017 17:52:58 +0200 Subject: [PATCH 097/297] Fix compilation issues after merge with master --- .../index/rankeval/DiscountedCumulativeGainTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 20f3b22e189..42c5ff113ff 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.Index; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -62,7 +62,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase { rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); + hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); assertEquals(13.84826362927298, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); @@ -114,7 +114,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase { } hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); + hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); EvalQueryQuality result = dcg.evaluate("id", hits, rated); @@ -171,7 +171,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase { for (int i = 0; i < 4; i++) { hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); + hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs); From 5a4124d4fbf8e186b9999c562e3496aca244ee66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 30 May 2017 15:21:02 +0200 Subject: [PATCH 098/297] Fixing template rendering after changes in master --- modules/rank-eval/build.gradle | 4 ---- .../index/rankeval/TransportRankEvalAction.java | 17 ++++++++--------- .../rank_eval/{10_basic.yaml => 10_basic.yml} | 0 .../test/rank_eval/{20_dcg.yaml => 20_dcg.yml} | 0 .../{30_failures.yaml => 30_failures.yml} | 0 ...TestRankEvalWithMustacheYAMLTestSuiteIT.java | 4 +--- 6 files changed, 9 insertions(+), 16 deletions(-) rename modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/{10_basic.yaml => 10_basic.yml} (100%) rename modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/{20_dcg.yaml => 20_dcg.yml} (100%) rename modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/{30_failures.yaml => 30_failures.yml} (100%) diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle index 1be16253b27..2c18fc40a08 100644 --- a/modules/rank-eval/build.gradle +++ b/modules/rank-eval/build.gradle @@ -22,7 +22,3 @@ esplugin { classname 'org.elasticsearch.index.rankeval.RankEvalPlugin' } -integTestCluster { - setting 'script.inline', 'true' - setting 'script.stored', 'true' -} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index eb775312b86..3891d4281db 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -26,19 +26,19 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.template.CompiledTemplate; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -97,10 +97,10 @@ public class TransportRankEvalAction ratedRequests.size()); Map errors = new ConcurrentHashMap<>(ratedRequests.size()); - Map scriptsWithoutParams = new HashMap<>(); + Map scriptsWithoutParams = new HashMap<>(); for (Entry entry : qualityTask.getTemplates().entrySet()) { scriptsWithoutParams.put(entry.getKey(), - scriptService.compile(entry.getValue(), ScriptContext.Standard.SEARCH)); + scriptService.compileTemplate(entry.getValue(), ExecutableScript.CONTEXT)); } for (RatedRequest ratedRequest : ratedRequests) { @@ -110,10 +110,9 @@ public class TransportRankEvalAction if (ratedSearchSource == null) { Map params = ratedRequest.getParams(); String templateId = ratedRequest.getTemplateId(); - CompiledScript compiled = scriptsWithoutParams.get(templateId); - BytesReference resolvedRequest = (BytesReference) (scriptService - .executable(compiled, params).run()); - try (XContentParser subParser = createParser(namedXContentRegistry, resolvedRequest, + CompiledTemplate compiled = scriptsWithoutParams.get(templateId); + String resolvedRequest = compiled.run(params); + try (XContentParser subParser = createParser(namedXContentRegistry, new BytesArray(resolvedRequest), XContentType.JSON)) { QueryParseContext parseContext = new QueryParseContext(subParser); ratedSearchSource = SearchSourceBuilder.fromXContent(parseContext); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml similarity index 100% rename from modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml rename to modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml similarity index 100% rename from modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml rename to modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml similarity index 100% rename from modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yaml rename to modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java index 401bddd3439..8132b5851e2 100644 --- a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java @@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import java.io.IOException; - public class SmokeTestRankEvalWithMustacheYAMLTestSuiteIT extends ESClientYamlSuiteTestCase { public SmokeTestRankEvalWithMustacheYAMLTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { @@ -34,7 +32,7 @@ public class SmokeTestRankEvalWithMustacheYAMLTestSuiteIT extends ESClientYamlSu } @ParametersFactory - public static Iterable parameters() throws IOException { + public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } From 37d0756d7a5e4a3762eb521bb1b5d655e1990ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 31 May 2017 11:41:10 +0200 Subject: [PATCH 099/297] Renaming rank-eval smoke test --- .../test/rank-eval/{30_template.yaml => 30_template.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/{30_template.yaml => 30_template.yml} (100%) diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yml similarity index 100% rename from qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml rename to qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yml From 4de4c795b7aa5fe547f83ed49912f6461dc2d12b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 14 Jun 2017 12:16:58 +0200 Subject: [PATCH 100/297] Fix issues after merging in master --- .../index/rankeval/RestRankEvalAction.java | 6 +++++- .../index/rankeval/TransportRankEvalAction.java | 11 +++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 1d721d69c09..2e6293213d4 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -150,7 +150,6 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; * */ public class RestRankEvalAction extends BaseRestHandler { - //private ScriptService scriptService; public RestRankEvalAction(Settings settings, RestController controller) { super(settings); @@ -189,4 +188,9 @@ public class RestRankEvalAction extends BaseRestHandler { rankEvalRequest.setRankEvalSpec(spec); } + + @Override + public String getName() { + return "rank_eval_action"; + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 3891d4281db..2be3ff4dadc 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -33,12 +33,11 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateScript; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.template.CompiledTemplate; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -97,10 +96,10 @@ public class TransportRankEvalAction ratedRequests.size()); Map errors = new ConcurrentHashMap<>(ratedRequests.size()); - Map scriptsWithoutParams = new HashMap<>(); + Map scriptsWithoutParams = new HashMap<>(); for (Entry entry : qualityTask.getTemplates().entrySet()) { scriptsWithoutParams.put(entry.getKey(), - scriptService.compileTemplate(entry.getValue(), ExecutableScript.CONTEXT)); + scriptService.compile(entry.getValue(), TemplateScript.CONTEXT)); } for (RatedRequest ratedRequest : ratedRequests) { @@ -110,8 +109,8 @@ public class TransportRankEvalAction if (ratedSearchSource == null) { Map params = ratedRequest.getParams(); String templateId = ratedRequest.getTemplateId(); - CompiledTemplate compiled = scriptsWithoutParams.get(templateId); - String resolvedRequest = compiled.run(params); + TemplateScript.Factory templateScript = scriptsWithoutParams.get(templateId); + String resolvedRequest = templateScript.newInstance(params).execute(); try (XContentParser subParser = createParser(namedXContentRegistry, new BytesArray(resolvedRequest), XContentType.JSON)) { QueryParseContext parseContext = new QueryParseContext(subParser); From 887ed68cf22d0b2a01a911c309007a9008aafc6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 14 Jul 2017 19:23:35 +0200 Subject: [PATCH 101/297] Fixing compilation issues and tests after merging in master --- .../index/rankeval/RatedRequest.java | 23 ++++--------------- .../rankeval/TransportRankEvalAction.java | 7 ++---- .../DiscountedCumulativeGainTests.java | 6 ++--- .../index/rankeval/PrecisionTests.java | 8 +++---- .../index/rankeval/RankEvalSpecTests.java | 20 ++++++---------- .../index/rankeval/RatedDocumentTests.java | 2 +- .../index/rankeval/RatedRequestsTests.java | 4 ++-- .../index/rankeval/ReciprocalRankTests.java | 6 ++--- 8 files changed, 27 insertions(+), 49 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index f673991a490..6d8bfd6485e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -22,14 +22,12 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; @@ -236,7 +234,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { private static final ParseField FIELDS_FIELD = new ParseField("summary_fields"); private static final ParseField TEMPLATE_ID_FIELD = new ParseField("template_id"); - private static final ConstructingObjectParser PARSER = + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("requests", a -> new RatedRequest((String) a[0], (List) a[1], (SearchSourceBuilder) a[2], (Map) a[3], (String) a[4])); @@ -246,20 +244,9 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> { return RatedDocument.fromXContent(p); }, RATINGS_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { - try { - return SearchSourceBuilder.fromXContent(c); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing request", ex); - } - }, REQUEST_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { - try { - return (Map) p.map(); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex); - } - }, PARAMS_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> + SearchSourceBuilder.fromXContent(p), REQUEST_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> (Map) p.map(), PARAMS_FIELD); PARSER.declareStringArray(RatedRequest::setSummaryFields, FIELDS_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TEMPLATE_ID_FIELD); } @@ -275,7 +262,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { * "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ] } */ public static RatedRequest fromXContent(XContentParser parser) { - return PARSER.apply(parser, new QueryParseContext(parser)); + return PARSER.apply(parser, null); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 2be3ff4dadc..3336faea7d0 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.TemplateScript; @@ -111,10 +110,8 @@ public class TransportRankEvalAction String templateId = ratedRequest.getTemplateId(); TemplateScript.Factory templateScript = scriptsWithoutParams.get(templateId); String resolvedRequest = templateScript.newInstance(params).execute(); - try (XContentParser subParser = createParser(namedXContentRegistry, new BytesArray(resolvedRequest), - XContentType.JSON)) { - QueryParseContext parseContext = new QueryParseContext(subParser); - ratedSearchSource = SearchSourceBuilder.fromXContent(parseContext); + try (XContentParser subParser = createParser(namedXContentRegistry, new BytesArray(resolvedRequest), XContentType.JSON)) { + ratedSearchSource = SearchSourceBuilder.fromXContent(subParser); } catch (IOException e) { listener.onFailure(e); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 42c5ff113ff..707e63b0af1 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -62,7 +62,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase { rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); + hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); assertEquals(13.84826362927298, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); @@ -114,7 +114,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase { } hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); + hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); EvalQueryQuality result = dcg.evaluate("id", hits, rated); @@ -171,7 +171,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase { for (int i = 0; i < 4; i++) { hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); + hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java index be6a9b89c39..91868b32082 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java @@ -126,7 +126,7 @@ public class PrecisionTests extends ESTestCase { // add an unlabeled search hit SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test", "testtype"), 3); searchHits[2] = new SearchHit(2, "2", new Text("testtype"), Collections.emptyMap()); - searchHits[2].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); + searchHits[2].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); EvalQueryQuality evaluated = (new Precision()).evaluate("id", searchHits, rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); @@ -148,7 +148,7 @@ public class PrecisionTests extends ESTestCase { SearchHit[] hits = new SearchHit[5]; for (int i = 0; i < 5; i++) { hits[i] = new SearchHit(i, i + "", new Text("type"), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0)); + hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); } EvalQueryQuality evaluated = (new Precision()).evaluate("id", hits, Collections.emptyList()); @@ -227,7 +227,7 @@ public class PrecisionTests extends ESTestCase { RankEvalTestHelper.copy(testItem, Precision::new)); } - private Precision mutateTestItem(Precision original) { + private static Precision mutateTestItem(Precision original) { boolean ignoreUnlabeled = original.getIgnoreUnlabeled(); int relevantThreshold = original.getRelevantRatingThreshold(); Precision precision = new Precision(); @@ -246,7 +246,7 @@ public class PrecisionTests extends ESTestCase { SearchHit[] hits = new SearchHit[rated.size()]; for (int i = 0; i < rated.size(); i++) { hits[i] = new SearchHit(i, i + "", new Text(type), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0)); + hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null)); } return hits; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index f8a879a74fb..75e72be2e12 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -68,24 +68,18 @@ public class RankEvalSpecTests extends ESTestCase { Collection templates = null; if (randomBoolean()) { - final Map params = randomBoolean() ? Collections.emptyMap() - : Collections.singletonMap("key", "value"); - ScriptType scriptType = randomFrom(ScriptType.values()); + final Map params = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("key", "value"); String script; - if (scriptType == ScriptType.INLINE) { - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - builder.startObject(); - builder.field("field", randomAlphaOfLengthBetween(1, 5)); - builder.endObject(); - script = builder.string(); - } - } else { - script = randomAlphaOfLengthBetween(1, 5); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + builder.startObject(); + builder.field("field", randomAlphaOfLengthBetween(1, 5)); + builder.endObject(); + script = builder.string(); } templates = new HashSet<>(); templates.add(new ScriptWithId("templateId", - new Script(scriptType, Script.DEFAULT_TEMPLATE_LANG, script, params))); + new Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, script, params))); Map templateParams = new HashMap<>(); templateParams.put("key", "value"); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index 3afbcef2a02..9e0d16620d0 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -66,7 +66,7 @@ public class RatedDocumentTests extends ESTestCase { RankEvalTestHelper.copy(testItem, RatedDocument::new)); } - public void testInvalidParsing() throws IOException { + public void testInvalidParsing() { expectThrows(IllegalArgumentException.class, () -> new RatedDocument(null, "abc", "abc", 10)); expectThrows(IllegalArgumentException.class, () -> new RatedDocument("", "abc", "abc", 10)); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 5d5ae92d4be..7467763508d 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -86,7 +86,7 @@ public class RatedRequestsTests extends ESTestCase { SearchSourceBuilder testRequest = null; if (randomBoolean() || forceRequest) { testRequest = new SearchSourceBuilder(); - testRequest.size(randomInt()); + testRequest.size(randomIntBetween(0, Integer.MAX_VALUE)); testRequest.query(new MatchAllQueryBuilder()); } else { int randomSize = randomIntBetween(1, 10); @@ -198,7 +198,7 @@ public class RatedRequestsTests extends ESTestCase { new NamedWriteableRegistry(namedWriteables))); } - private RatedRequest mutateTestItem(RatedRequest original) { + private static RatedRequest mutateTestItem(RatedRequest original) { String id = original.getId(); SearchSourceBuilder testRequest = original.getTestRequest(); List ratedDocs = original.getRatedDocs(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 60eb6325a9a..4d0f3347acd 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -156,12 +156,12 @@ public class ReciprocalRankTests extends ESTestCase { SearchHit[] hits = new SearchHit[to + 1 - from]; for (int i = from; i <= to; i++) { hits[i] = new SearchHit(i, i + "", new Text(type), Collections.emptyMap()); - hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0)); + hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null)); } return hits; } - private ReciprocalRank createTestItem() { + private static ReciprocalRank createTestItem() { ReciprocalRank testItem = new ReciprocalRank(); testItem.setRelevantRatingThreshhold(randomIntBetween(0, 20)); return testItem; @@ -182,7 +182,7 @@ public class ReciprocalRankTests extends ESTestCase { RankEvalTestHelper.copy(testItem, ReciprocalRank::new)); } - private ReciprocalRank mutateTestItem(ReciprocalRank testItem) { + private static ReciprocalRank mutateTestItem(ReciprocalRank testItem) { int relevantThreshold = testItem.getRelevantRatingThreshold(); ReciprocalRank rank = new ReciprocalRank(); rank.setRelevantRatingThreshhold( From bc544e2d1bf15faba1400eff0d7ba24258f62399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 23 Aug 2017 12:05:52 +0200 Subject: [PATCH 102/297] Adapt branch to changes on master --- .../java/org/elasticsearch/index/rankeval/DocumentKey.java | 4 ++-- .../java/org/elasticsearch/index/rankeval/RankEvalSpec.java | 4 ++-- .../java/org/elasticsearch/index/rankeval/RatedDocument.java | 4 ++-- .../java/org/elasticsearch/index/rankeval/RatedRequest.java | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java index 44b0aa21359..bb032871e48 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java @@ -19,17 +19,17 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; import java.util.Objects; -public class DocumentKey extends ToXContentToBytes implements Writeable { +public class DocumentKey implements Writeable, ToXContentObject { private String docId; private String type; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 5c7a713f23b..a9f8bed08a0 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -19,13 +19,13 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.script.Script; @@ -47,7 +47,7 @@ import java.util.Objects; * search queries. */ -public class RankEvalSpec extends ToXContentToBytes implements Writeable { +public class RankEvalSpec implements Writeable, ToXContentObject { /** * Collection of query specifications, that is e.g. search request templates * to use for query translation. diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index 0102de14436..5b1d24ff3d7 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -19,12 +19,12 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -34,7 +34,7 @@ import java.util.Objects; /** * A document ID and its rating for the query QA use case. * */ -public class RatedDocument extends ToXContentToBytes implements Writeable { +public class RatedDocument implements Writeable, ToXContentObject { public static final ParseField RATING_FIELD = new ParseField("rating"); public static final ParseField DOC_ID_FIELD = new ParseField("_id"); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 6d8bfd6485e..85350c1dc04 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -19,13 +19,13 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -50,7 +50,7 @@ import java.util.Set; * in the set of rated documents as part of a QAQuery. */ @SuppressWarnings("unchecked") -public class RatedRequest extends ToXContentToBytes implements Writeable { +public class RatedRequest implements Writeable, ToXContentObject { private String id; private List indices = new ArrayList<>(); private List types = new ArrayList<>(); From 56360ecfb51fbce93cca805b65f7bc3da7ff6564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 23 Aug 2017 12:22:07 +0200 Subject: [PATCH 103/297] Fix failing tests due to xContent changes --- .../org/elasticsearch/index/rankeval/DocumentKey.java | 5 +++++ .../org/elasticsearch/index/rankeval/RankEvalSpec.java | 6 ++++++ .../org/elasticsearch/index/rankeval/RatedDocument.java | 6 ++++++ .../org/elasticsearch/index/rankeval/RatedRequest.java | 6 ++++++ .../elasticsearch/index/rankeval/RatedRequestsTests.java | 8 ++++---- 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java index bb032871e48..c6643fa8e4a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java @@ -116,4 +116,9 @@ public class DocumentKey implements Writeable, ToXContentObject { builder.endObject(); return builder; } + + @Override + public String toString() { + return Strings.toString(this); + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index a9f8bed08a0..60d0a5dd4fd 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -239,6 +240,11 @@ public class RankEvalSpec implements Writeable, ToXContentObject { return builder; } + @Override + public String toString() { + return Strings.toString(this); + } + @Override public final boolean equals(Object obj) { if (this == obj) { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index 5b1d24ff3d7..e78a7f79e2f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -110,6 +111,11 @@ public class RatedDocument implements Writeable, ToXContentObject { return builder; } + @Override + public String toString() { + return Strings.toString(this); + } + @Override public final boolean equals(Object obj) { if (this == obj) { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 85350c1dc04..d55d0cd322f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -296,6 +297,11 @@ public class RatedRequest implements Writeable, ToXContentObject { return builder; } + @Override + public String toString() { + return Strings.toString(this); + } + @Override public final boolean equals(Object obj) { if (this == obj) { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 7467763508d..f046f20a30d 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -268,8 +268,8 @@ public class RatedRequestsTests extends ESTestCase { IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder())); assertEquals( - "Found duplicate rated document key [{ \"_index\" : \"index1\", " - + "\"_type\" : \"type1\", \"_id\" : \"id1\"}]", + "Found duplicate rated document key [{\"_index\":\"index1\"," + + "\"_type\":\"type1\",\"_id\":\"id1\"}]", ex.getMessage()); // templated path, no summary fields Map params = new HashMap<>(); @@ -277,8 +277,8 @@ public class RatedRequestsTests extends ESTestCase { ex = expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, params, "templateId")); assertEquals( - "Found duplicate rated document key [{ \"_index\" : \"index1\", " - + "\"_type\" : \"type1\", \"_id\" : \"id1\"}]", + "Found duplicate rated document key [{\"_index\":\"index1\"," + + "\"_type\":\"type1\",\"_id\":\"id1\"}]", ex.getMessage()); } From cb4fd3bac66fbaeab62aa25fc46ba78da1a378fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 23 Aug 2017 12:30:09 +0200 Subject: [PATCH 104/297] Fix more tests --- .../org/elasticsearch/index/rankeval/RankEvalRequestIT.java | 2 +- .../org/elasticsearch/index/rankeval/RatedRequestsTests.java | 2 +- .../test/resources/rest-api-spec/test/rank_eval/30_failures.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 49d5db32919..a043d90564f 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -147,7 +147,7 @@ public class RankEvalRequestIT extends ESIntegTestCase { specifications.add(amsterdamRequest); SearchSourceBuilder brokenQuery = new SearchSourceBuilder(); - RangeQueryBuilder brokenRangeQuery = new RangeQueryBuilder("text").timeZone("CET"); + RangeQueryBuilder brokenRangeQuery = new RangeQueryBuilder("text").timeZone("CET").from("Basel").to("Zehlendorf"); brokenQuery.query(brokenRangeQuery); RatedRequest brokenRequest = new RatedRequest("broken_query", createRelevant("1"), brokenQuery); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index f046f20a30d..3b7d64ff4c7 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -215,7 +215,7 @@ public class RatedRequestsTests extends ESTestCase { break; case 1: if (testRequest != null) { - int size = randomValueOtherThan(testRequest.size(), () -> randomInt()); + int size = randomValueOtherThan(testRequest.size(), () -> randomInt(Integer.MAX_VALUE)); testRequest = new SearchSourceBuilder(); testRequest.size(size); testRequest.query(new MatchAllQueryBuilder()); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml index 1d19a94cdac..f3b7bf4627e 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml @@ -23,7 +23,7 @@ }, { "id" : "invalid_query", - "request": { "query": { "range" : { "bar" : { "time_zone": "+01:00" }}}}, + "request": { "query": { "range" : { "bar" : { "from" : "Basel", "time_zone": "+01:00" }}}}, "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] } ], From c83ec1f1335ebd4323a41f95b55818f8cf6e43c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 15 Sep 2017 13:44:40 +0200 Subject: [PATCH 105/297] Fixing test after merging in master --- .../index/rankeval/RankEvalRequestIT.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index a043d90564f..fdae133afc2 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -21,7 +21,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.rankeval.PrecisionTests.Rating; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -54,12 +54,12 @@ public class RankEvalRequestIT extends ESIntegTestCase { ensureGreen(); client().prepareIndex("test", "testtype").setId("1") - .setSource("text", "berlin", "title", "Berlin, Germany").get(); - client().prepareIndex("test", "testtype").setId("2").setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("3").setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("4").setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("5").setSource("text", "amsterdam").get(); - client().prepareIndex("test", "testtype").setId("6").setSource("text", "amsterdam").get(); + .setSource("text", "berlin", "title", "Berlin, Germany", "population", 3670622).get(); + client().prepareIndex("test", "testtype").setId("2").setSource("text", "amsterdam", "population", 851573).get(); + client().prepareIndex("test", "testtype").setId("3").setSource("text", "amsterdam", "population", 851573).get(); + client().prepareIndex("test", "testtype").setId("4").setSource("text", "amsterdam", "population", 851573).get(); + client().prepareIndex("test", "testtype").setId("5").setSource("text", "amsterdam", "population", 851573).get(); + client().prepareIndex("test", "testtype").setId("6").setSource("text", "amsterdam", "population", 851573).get(); refresh(); } @@ -147,8 +147,7 @@ public class RankEvalRequestIT extends ESIntegTestCase { specifications.add(amsterdamRequest); SearchSourceBuilder brokenQuery = new SearchSourceBuilder(); - RangeQueryBuilder brokenRangeQuery = new RangeQueryBuilder("text").timeZone("CET").from("Basel").to("Zehlendorf"); - brokenQuery.query(brokenRangeQuery); + brokenQuery.query(QueryBuilders.termQuery("population", "noStringOnNumericFields")); RatedRequest brokenRequest = new RatedRequest("broken_query", createRelevant("1"), brokenQuery); brokenRequest.setIndices(indices); @@ -166,8 +165,8 @@ public class RankEvalRequestIT extends ESIntegTestCase { assertEquals(1, response.getFailures().size()); ElasticsearchException[] rootCauses = ElasticsearchException .guessRootCauses(response.getFailures().get("broken_query")); - assertEquals("[range] time_zone can not be applied to non date field [text]", - rootCauses[0].getMessage()); + assertEquals("java.lang.NumberFormatException: For input string: \"noStringOnNumericFields\"", + rootCauses[0].getCause().toString()); } From 0a6c6ac360063f2e0b716fcba464c176f42ad9d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 6 Nov 2017 22:46:59 +0100 Subject: [PATCH 106/297] Remove usage of types in rank_eval endpoint --- .../index/rankeval/DocumentKey.java | 24 +-- .../rankeval/RankedListQualityMetric.java | 43 ++--- .../index/rankeval/RatedDocument.java | 21 +-- .../index/rankeval/RatedRequest.java | 24 +-- .../index/rankeval/RestRankEvalAction.java | 13 +- .../rankeval/TransportRankEvalAction.java | 4 - .../DiscountedCumulativeGainTests.java | 121 ++++++------- .../index/rankeval/DocumentKeyTests.java | 18 +- .../index/rankeval/PrecisionTests.java | 111 +++++------- .../index/rankeval/RankEvalRequestIT.java | 8 +- .../index/rankeval/RankEvalSpecTests.java | 76 +++------ .../index/rankeval/RankEvalTestHelper.java | 1 + .../index/rankeval/RatedDocumentTests.java | 30 +--- .../index/rankeval/RatedRequestsTests.java | 160 ++++++------------ .../index/rankeval/ReciprocalRankTests.java | 62 +++---- .../rest-api-spec/test/rank_eval/10_basic.yml | 22 +-- .../rest-api-spec/test/rank_eval/20_dcg.yml | 48 +++--- .../test/rank_eval/30_failures.yml | 4 +- .../smoketest/SmokeMultipleTemplatesIT.java | 9 +- .../test/rank-eval/30_template.yml | 10 +- 20 files changed, 284 insertions(+), 525 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java index c6643fa8e4a..7bbda60f18c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java @@ -32,40 +32,30 @@ import java.util.Objects; public class DocumentKey implements Writeable, ToXContentObject { private String docId; - private String type; private String index; void setIndex(String index) { this.index = index; } - void setType(String type) { - this.type = type; - } - void setDocId(String docId) { this.docId = docId; } - public DocumentKey(String index, String type, String docId) { + public DocumentKey(String index, String docId) { if (Strings.isNullOrEmpty(index)) { throw new IllegalArgumentException("Index must be set for each rated document"); } - if(Strings.isNullOrEmpty(type)) { - throw new IllegalArgumentException("Type must be set for each rated document"); - } if (Strings.isNullOrEmpty(docId)) { throw new IllegalArgumentException("DocId must be set for each rated document"); } this.index = index; - this.type = type; this.docId = docId; } public DocumentKey(StreamInput in) throws IOException { this.index = in.readString(); - this.type = in.readString(); this.docId = in.readString(); } @@ -73,10 +63,6 @@ public class DocumentKey implements Writeable, ToXContentObject { return index; } - public String getType() { - return type; - } - public String getDocID() { return docId; } @@ -84,7 +70,6 @@ public class DocumentKey implements Writeable, ToXContentObject { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(index); - out.writeString(type); out.writeString(docId); } @@ -97,21 +82,18 @@ public class DocumentKey implements Writeable, ToXContentObject { return false; } DocumentKey other = (DocumentKey) obj; - return Objects.equals(index, other.index) && - Objects.equals(type, other.type) && - Objects.equals(docId, other.docId); + return Objects.equals(index, other.index) && Objects.equals(docId, other.docId); } @Override public final int hashCode() { - return Objects.hash(index, type, docId); + return Objects.hash(index, docId); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), index); - builder.field(RatedDocument.TYPE_FIELD.getPreferredName(), type); builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), docId); builder.endObject(); return builder; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index 15d4eb03064..cf1da4388fe 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -35,29 +35,23 @@ import java.util.Optional; import java.util.stream.Collectors; /** - * Classes implementing this interface provide a means to compute the quality of - * a result list returned by some search. - * - * RelevancyLevel specifies the type of object determining the relevancy level - * of some known docid. + * Classes implementing this interface provide a means to compute the quality of a result list returned by some search. */ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { /** - * Returns a single metric representing the ranking quality of a set of - * returned documents wrt. to a set of document Ids labeled as relevant for - * this search. + * Returns a single metric representing the ranking quality of a set of returned + * documents wrt. to a set of document Ids labeled as relevant for this search. * * @param taskId - * the id of the query for which the ranking is currently - * evaluated + * the id of the query for which the ranking is currently evaluated * @param hits * the result hits as returned by a search request * @param ratedDocs - * the documents that were ranked by human annotators for this - * query case - * @return some metric representing the quality of the result hit list wrt. - * to relevant doc ids. + * the documents that were ranked by human annotators for this query + * case + * @return some metric representing the quality of the result hit list wrt. to + * relevant doc ids. */ EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs); @@ -65,8 +59,7 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { RankedListQualityMetric rc; Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { - throw new ParsingException(parser.getTokenLocation(), - "[_na] missing required metric name"); + throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name"); } String metricName = parser.currentName(); @@ -82,8 +75,7 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { rc = DiscountedCumulativeGain.fromXContent(parser); break; default: - throw new ParsingException(parser.getTokenLocation(), - "[_na] unknown query metric name [{}]", metricName); + throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName); } if (parser.currentToken() == XContentParser.Token.END_OBJECT) { // if we are at END_OBJECT, move to the next one... @@ -92,14 +84,13 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { return rc; } - static List joinHitsWithRatings(SearchHit[] hits, - List ratedDocs) { + static List joinHitsWithRatings(SearchHit[] hits, List ratedDocs) { // join hits with rated documents Map ratedDocumentMap = ratedDocs.stream() .collect(Collectors.toMap(RatedDocument::getKey, item -> item)); List ratedSearchHits = new ArrayList<>(hits.length); for (SearchHit hit : hits) { - DocumentKey key = new DocumentKey(hit.getIndex(), hit.getType(), hit.getId()); + DocumentKey key = new DocumentKey(hit.getIndex(), hit.getId()); RatedDocument ratedDoc = ratedDocumentMap.get(key); if (ratedDoc != null) { ratedSearchHits.add(new RatedSearchHit(hit, Optional.of(ratedDoc.getRating()))); @@ -112,16 +103,12 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { static List filterUnknownDocuments(List ratedHits) { // join hits with rated documents - List unknownDocs = ratedHits.stream() - .filter(hit -> hit.getRating().isPresent() == false) - .map(hit -> new DocumentKey(hit.getSearchHit().getIndex(), - hit.getSearchHit().getType(), hit.getSearchHit().getId())) - .collect(Collectors.toList()); + List unknownDocs = ratedHits.stream().filter(hit -> hit.getRating().isPresent() == false) + .map(hit -> new DocumentKey(hit.getSearchHit().getIndex(), hit.getSearchHit().getId())).collect(Collectors.toList()); return unknownDocs; } default double combine(Collection partialResults) { - return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() - / partialResults.size(); + return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size(); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index e78a7f79e2f..ec19aa8aa0e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -34,21 +34,18 @@ import java.util.Objects; /** * A document ID and its rating for the query QA use case. - * */ + */ public class RatedDocument implements Writeable, ToXContentObject { public static final ParseField RATING_FIELD = new ParseField("rating"); public static final ParseField DOC_ID_FIELD = new ParseField("_id"); - public static final ParseField TYPE_FIELD = new ParseField("_type"); public static final ParseField INDEX_FIELD = new ParseField("_index"); - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("rated_document", - a -> new RatedDocument((String) a[0], (String) a[1], (String) a[2], (Integer) a[3])); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rated_document", + a -> new RatedDocument((String) a[0], (String) a[1], (Integer) a[2])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_FIELD); - PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), DOC_ID_FIELD); PARSER.declareInt(ConstructingObjectParser.constructorArg(), RATING_FIELD); } @@ -56,8 +53,8 @@ public class RatedDocument implements Writeable, ToXContentObject { private int rating; private DocumentKey key; - public RatedDocument(String index, String type, String docId, int rating) { - this(new DocumentKey(index, type, docId), rating); + public RatedDocument(String index, String docId, int rating) { + this(new DocumentKey(index, docId), rating); } public RatedDocument(StreamInput in) throws IOException { @@ -78,10 +75,6 @@ public class RatedDocument implements Writeable, ToXContentObject { return key.getIndex(); } - public String getType() { - return key.getType(); - } - public String getDocID() { return key.getDocID(); } @@ -104,7 +97,6 @@ public class RatedDocument implements Writeable, ToXContentObject { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(INDEX_FIELD.getPreferredName(), key.getIndex()); - builder.field(TYPE_FIELD.getPreferredName(), key.getType()); builder.field(DOC_ID_FIELD.getPreferredName(), key.getDocID()); builder.field(RATING_FIELD.getPreferredName(), rating); builder.endObject(); @@ -125,8 +117,7 @@ public class RatedDocument implements Writeable, ToXContentObject { return false; } RatedDocument other = (RatedDocument) obj; - return Objects.equals(key, other.key) && - Objects.equals(rating, other.rating); + return Objects.equals(key, other.key) && Objects.equals(rating, other.rating); } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index d55d0cd322f..af322add5ec 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -54,7 +54,6 @@ import java.util.Set; public class RatedRequest implements Writeable, ToXContentObject { private String id; private List indices = new ArrayList<>(); - private List types = new ArrayList<>(); private List summaryFields = new ArrayList<>(); /** Collection of rated queries for this query QA specification. */ private List ratedDocs = new ArrayList<>(); @@ -93,7 +92,7 @@ public class RatedRequest implements Writeable, ToXContentObject { "If template parameters are supplied need to set id of template to apply " + "them to too."); } - // No documents with same _index/_type/id allowed. + // No documents with same _index/id allowed. Set docKeys = new HashSet<>(); for (RatedDocument doc : ratedDocs) { if (docKeys.add(doc.getKey()) == false) { @@ -131,11 +130,6 @@ public class RatedRequest implements Writeable, ToXContentObject { for (int i = 0; i < indicesSize; i++) { this.indices.add(in.readString()); } - int typesSize = in.readInt(); - types = new ArrayList<>(typesSize); - for (int i = 0; i < typesSize; i++) { - this.types.add(in.readString()); - } int intentSize = in.readInt(); ratedDocs = new ArrayList<>(intentSize); for (int i = 0; i < intentSize; i++) { @@ -159,10 +153,6 @@ public class RatedRequest implements Writeable, ToXContentObject { for (String index : indices) { out.writeString(index); } - out.writeInt(types.size()); - for (String type : types) { - out.writeString(type); - } out.writeInt(ratedDocs.size()); for (RatedDocument ratedDoc : ratedDocs) { ratedDoc.writeTo(out); @@ -187,14 +177,6 @@ public class RatedRequest implements Writeable, ToXContentObject { return indices; } - public void setTypes(List types) { - this.types = types; - } - - public List getTypes() { - return types; - } - /** Returns a user supplied spec id for easier referencing. */ public String getId() { return id; @@ -314,7 +296,7 @@ public class RatedRequest implements Writeable, ToXContentObject { RatedRequest other = (RatedRequest) obj; return Objects.equals(id, other.id) && Objects.equals(testRequest, other.testRequest) - && Objects.equals(indices, other.indices) && Objects.equals(types, other.types) + && Objects.equals(indices, other.indices) && Objects.equals(summaryFields, other.summaryFields) && Objects.equals(ratedDocs, other.ratedDocs) && Objects.equals(params, other.params) @@ -323,7 +305,7 @@ public class RatedRequest implements Writeable, ToXContentObject { @Override public final int hashCode() { - return Objects.hash(id, testRequest, indices, types, summaryFields, ratedDocs, params, + return Objects.hash(id, testRequest, indices, summaryFields, ratedDocs, params, templateId); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 2e6293213d4..5edd1061936 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -135,13 +135,13 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; "quality_level": 0.4, "unknown_docs": { "amsterdam_query": [ - { "index" : "test", "type" : "my_type", "doc_id" : "21"}, - { "index" : "test", "type" : "my_type", "doc_id" : "5"}, - { "index" : "test", "type" : "my_type", "doc_id" : "9"} + { "index" : "test", "doc_id" : "21"}, + { "index" : "test", "doc_id" : "5"}, + { "index" : "test", "doc_id" : "9"} ] }, { "berlin_query": [ - { "index" : "test", "type" : "my_type", "doc_id" : "42"} + { "index" : "test", "doc_id" : "42"} ] } }] @@ -157,8 +157,6 @@ public class RestRankEvalAction extends BaseRestHandler { controller.registerHandler(POST, "/_rank_eval", this); controller.registerHandler(GET, "/{index}/_rank_eval", this); controller.registerHandler(POST, "/{index}/_rank_eval", this); - controller.registerHandler(GET, "/{index}/{type}/_rank_eval", this); - controller.registerHandler(POST, "/{index}/{type}/_rank_eval", this); } @Override @@ -176,13 +174,10 @@ public class RestRankEvalAction extends BaseRestHandler { XContentParser parser) { List indices = Arrays .asList(Strings.splitStringByCommaToArray(request.param("index"))); - List types = Arrays - .asList(Strings.splitStringByCommaToArray(request.param("type"))); RankEvalSpec spec = null; spec = RankEvalSpec.parse(parser); for (RatedRequest specification : spec.getRatedRequests()) { specification.setIndices(indices); - specification.setTypes(types); } ; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 3336faea7d0..8c9024d0500 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -128,10 +128,6 @@ public class TransportRankEvalAction indices = ratedRequest.getIndices().toArray(indices); SearchRequest templatedRequest = new SearchRequest(indices, ratedSearchSource); - String[] types = new String[ratedRequest.getTypes().size()]; - types = ratedRequest.getTypes().toArray(types); - templatedRequest.types(types); - RequestTask task = new RequestTask(templatedRequest, searchListener); taskQueue.add(task); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 707e63b0af1..4154000a7b0 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -43,13 +43,12 @@ public class DiscountedCumulativeGainTests extends ESTestCase { /** * Assuming the docs are ranked in the following order: * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) - * / log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / + * log_2(rank + 1) * ------------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 - * | 1.8927892607143721 3 | 3 | 7.0 | 2.0 | 3.5 4 | 0 | 0.0 - * | 2.321928094887362 | 0.0 5 | 1 | 1.0 | 2.584962500721156 - * | 0.38685280723454163 6 | 2 | 3.0 | 2.807354922057604 + * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 + * 3 | 3 | 7.0 | 2.0 | 3.5 4 | 0 | 0.0 | 2.321928094887362 | 0.0 5 | 1 | 1.0 + * | 2.584962500721156 | 0.38685280723454163 6 | 2 | 3.0 | 2.807354922057604 * | 1.0686215613240666 * * dcg = 13.84826362927298 (sum of last column) @@ -59,45 +58,41 @@ public class DiscountedCumulativeGainTests extends ESTestCase { int[] relevanceRatings = new int[] { 3, 2, 3, 0, 1, 2 }; SearchHit[] hits = new SearchHit[6]; for (int i = 0; i < 6; i++) { - rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); - hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), - Collections.emptyMap()); + rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); + hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); assertEquals(13.84826362927298, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); /** - * Check with normalization: to get the maximal possible dcg, sort - * documents by relevance in descending order + * Check with normalization: to get the maximal possible dcg, sort documents by + * relevance in descending order * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - * - 1) / log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / + * log_2(rank + 1) * --------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0  | 7.0 2 | 3 | 7.0 | - * 1.5849625007211563 | 4.416508275000202 3 | 2 | 3.0 | 2.0  | 1.5 4 | 2 - * | 3.0 | 2.321928094887362  | 1.2920296742201793 5 | 1 | 1.0 | - * 2.584962500721156  | 0.38685280723454163 6 | 0 | 0.0 | - * 2.807354922057604  | 0.0 + * 1 | 3 | 7.0 | 1.0  | 7.0 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 + * 3 | 2 | 3.0 | 2.0  | 1.5 4 | 2 | 3.0 | 2.321928094887362  + * | 1.2920296742201793 5 | 1 | 1.0 | 2.584962500721156  | 0.38685280723454163 6 + * | 0 | 0.0 | 2.807354922057604  | 0.0 * * idcg = 14.595390756454922 (sum of last column) */ dcg.setNormalize(true); - assertEquals(13.84826362927298 / 14.595390756454922, - dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); + assertEquals(13.84826362927298 / 14.595390756454922, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); } /** * This tests metric when some documents in the search result don't have a * rating provided by the user. * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) - * / log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / + * log_2(rank + 1) * ------------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 - * | 1.8927892607143721 3 | 3 | 7.0 | 2.0 | 3.5 4 | n/a | n/a | n/a | n/a 5 - * | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163 6 | n/a | n/a | n/a | - * n/a + * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 + * 3 | 3 | 7.0 | 2.0 | 3.5 4 | n/a | n/a | n/a | n/a 5 | 1 | 1.0 + * | 2.584962500721156 | 0.38685280723454163 6 | n/a | n/a | n/a | n/a * * dcg = 12.779642067948913 (sum of last column) */ @@ -108,12 +103,10 @@ public class DiscountedCumulativeGainTests extends ESTestCase { for (int i = 0; i < 6; i++) { if (i < relevanceRatings.length) { if (relevanceRatings[i] != null) { - rated.add(new RatedDocument("index", "type", Integer.toString(i), - relevanceRatings[i])); + rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); } } - hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), - Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); @@ -122,36 +115,34 @@ public class DiscountedCumulativeGainTests extends ESTestCase { assertEquals(2, filterUnknownDocuments(result.getHitsAndRatings()).size()); /** - * Check with normalization: to get the maximal possible dcg, sort - * documents by relevance in descending order + * Check with normalization: to get the maximal possible dcg, sort documents by + * relevance in descending order * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - * - 1) / log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / + * log_2(rank + 1) * ---------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0  | 7.0 2 | 3 | 7.0 | - * 1.5849625007211563 | 4.416508275000202 3 | 2 | 3.0 | 2.0  | 1.5 4 | 1 - * | 1.0 | 2.321928094887362   | 0.43067655807339 5 | n.a | n.a | n.a.  - * | n.a. 6 | n.a | n.a | n.a  | n.a + * 1 | 3 | 7.0 | 1.0  | 7.0 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 + * 3 | 2 | 3.0 | 2.0  | 1.5 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339 + * 5 | n.a | n.a | n.a.  | n.a. 6 | n.a | n.a | n.a  | n.a * * idcg = 13.347184833073591 (sum of last column) */ dcg.setNormalize(true); - assertEquals(12.779642067948913 / 13.347184833073591, - dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); + assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); } /** * This tests that normalization works as expected when there are more rated - * documents than search hits because we restrict DCG to be calculated at - * the fourth position + * documents than search hits because we restrict DCG to be calculated at the + * fourth position * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) - * / log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / + * log_2(rank + 1) * ------------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 - * | 1.8927892607143721 3 | 3 | 7.0 | 2.0 | 3.5 4 | n/a | n/a | n/a | n/a - * ----------------------------------------------------------------- 5 | 1 - * | 1.0 | 2.584962500721156 | 0.38685280723454163 6 | n/a | n/a | n/a | n/a + * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 + * 3 | 3 | 7.0 | 2.0 | 3.5 4 | n/a | n/a | n/a | n/a + * ----------------------------------------------------------------- 5 | 1 | 1.0 + * | 2.584962500721156 | 0.38685280723454163 6 | n/a | n/a | n/a | n/a * * dcg = 12.392789260714371 (sum of last column until position 4) */ @@ -161,16 +152,14 @@ public class DiscountedCumulativeGainTests extends ESTestCase { for (int i = 0; i < 6; i++) { if (i < relevanceRatings.length) { if (relevanceRatings[i] != null) { - ratedDocs.add(new RatedDocument("index", "type", Integer.toString(i), - relevanceRatings[i])); + ratedDocs.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); } } } // only create four hits SearchHit[] hits = new SearchHit[4]; for (int i = 0; i < 4; i++) { - hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), - Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); @@ -179,28 +168,25 @@ public class DiscountedCumulativeGainTests extends ESTestCase { assertEquals(1, filterUnknownDocuments(result.getHitsAndRatings()).size()); /** - * Check with normalization: to get the maximal possible dcg, sort - * documents by relevance in descending order + * Check with normalization: to get the maximal possible dcg, sort documents by + * relevance in descending order * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - * - 1) / log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / + * log_2(rank + 1) * --------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0  | 7.0 2 | 3 | 7.0 | - * 1.5849625007211563 | 4.416508275000202 3 | 2 | 3.0 | 2.0  | 1.5 4 | 1 - * | 1.0 | 2.321928094887362   | 0.43067655807339 + * 1 | 3 | 7.0 | 1.0  | 7.0 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 + * 3 | 2 | 3.0 | 2.0  | 1.5 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339 * --------------------------------------------------------------------------------------- * 5 | n.a | n.a | n.a.  | n.a. 6 | n.a | n.a | n.a  | n.a * * idcg = 13.347184833073591 (sum of last column) */ dcg.setNormalize(true); - assertEquals(12.392789260714371 / 13.347184833073591, - dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), 0.00001); + assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), 0.00001); } public void testParseFromXContent() throws IOException { - String xContent = " {\n" + " \"unknown_doc_rating\": 2,\n" + " \"normalize\": true\n" - + "}"; + String xContent = " {\n" + " \"unknown_doc_rating\": 2,\n" + " \"normalize\": true\n" + "}"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) { DiscountedCumulativeGain dcgAt = DiscountedCumulativeGain.fromXContent(parser); assertEquals(2, dcgAt.getUnknownDocRating().intValue()); @@ -218,8 +204,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { DiscountedCumulativeGain testItem = createTestItem(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - XContentBuilder shuffled = shuffleXContent( - testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); itemParser.nextToken(); @@ -232,8 +217,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase { public void testSerialization() throws IOException { DiscountedCumulativeGain original = createTestItem(); - DiscountedCumulativeGain deserialized = RankEvalTestHelper.copy(original, - DiscountedCumulativeGain::new); + DiscountedCumulativeGain deserialized = RankEvalTestHelper.copy(original, DiscountedCumulativeGain::new); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); @@ -254,8 +238,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase { List mutators = new ArrayList<>(); mutators.add(() -> gain.setNormalize(!original.getNormalize())); - mutators.add(() -> gain.setUnknownDocRating( - randomValueOtherThan(unknownDocRating, () -> randomIntBetween(0, 10)))); + mutators.add(() -> gain.setUnknownDocRating(randomValueOtherThan(unknownDocRating, () -> randomIntBetween(0, 10)))); randomFrom(mutators).run(); return gain; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java index 1b3efa996c8..7a241451bc9 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java @@ -23,14 +23,12 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; - public class DocumentKeyTests extends ESTestCase { static DocumentKey createRandomRatedDocumentKey() { String index = randomAlphaOfLengthBetween(1, 10); - String type = randomAlphaOfLengthBetween(1, 10); String docId = randomAlphaOfLengthBetween(1, 10); - return new DocumentKey(index, type, docId); + return new DocumentKey(index, docId); } public DocumentKey createTestItem() { @@ -39,28 +37,24 @@ public class DocumentKeyTests extends ESTestCase { public DocumentKey mutateTestItem(DocumentKey original) { String index = original.getIndex(); - String type = original.getType(); String docId = original.getDocID(); - switch (randomIntBetween(0, 2)) { + switch (randomIntBetween(0, 1)) { case 0: index = index + "_"; break; case 1: - type = type + "_"; - break; - case 2: docId = docId + "_"; break; default: - throw new IllegalStateException("The test should only allow three parameters mutated"); + throw new IllegalStateException("The test should only allow two parameters mutated"); } - return new DocumentKey(index, type, docId); + return new DocumentKey(index, docId); } public void testEqualsAndHash() throws IOException { DocumentKey testItem = createRandomRatedDocumentKey(); RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - new DocumentKey(testItem.getIndex(), testItem.getType(), testItem.getDocID())); + new DocumentKey(testItem.getIndex(), testItem.getDocID())); } public void testSerialization() throws IOException { @@ -70,6 +64,4 @@ public class DocumentKeyTests extends ESTestCase { assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); } - - } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java index 91868b32082..a520de378b9 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java @@ -42,96 +42,72 @@ public class PrecisionTests extends ESTestCase { public void testPrecisionAtFiveCalculation() { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - EvalQueryQuality evaluated = (new Precision()).evaluate("id", - toSearchHits(rated, "test", "testtype"), rated); + rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal())); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated, "test"), rated); assertEquals(1, evaluated.getQualityLevel(), 0.00001); - assertEquals(1, - ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(1, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(1, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveIgnoreOneResult() { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); - EvalQueryQuality evaluated = (new Precision()).evaluate("id", - toSearchHits(rated, "test", "testtype"), rated); + rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "2", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "3", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "4", Rating.IRRELEVANT.ordinal())); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated, "test"), rated); assertEquals((double) 4 / 5, evaluated.getQualityLevel(), 0.00001); - assertEquals(4, - ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(4, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } /** - * test that the relevant rating threshold can be set to something larger - * than 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant, - * doc 3 and 4 to be relevant + * test that the relevant rating threshold can be set to something larger than + * 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant, doc 3 and + * 4 to be relevant */ public void testPrecisionAtFiveRelevanceThreshold() { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "testtype", "0", 0)); - rated.add(new RatedDocument("test", "testtype", "1", 1)); - rated.add(new RatedDocument("test", "testtype", "2", 2)); - rated.add(new RatedDocument("test", "testtype", "3", 3)); - rated.add(new RatedDocument("test", "testtype", "4", 4)); + rated.add(new RatedDocument("test", "0", 0)); + rated.add(new RatedDocument("test", "1", 1)); + rated.add(new RatedDocument("test", "2", 2)); + rated.add(new RatedDocument("test", "3", 3)); + rated.add(new RatedDocument("test", "4", 4)); Precision precisionAtN = new Precision(); precisionAtN.setRelevantRatingThreshhold(2); - EvalQueryQuality evaluated = precisionAtN.evaluate("id", - toSearchHits(rated, "test", "testtype"), rated); + EvalQueryQuality evaluated = precisionAtN.evaluate("id", toSearchHits(rated, "test"), rated); assertEquals((double) 3 / 5, evaluated.getQualityLevel(), 0.00001); - assertEquals(3, - ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveCorrectIndex() { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test_other", "testtype", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test_other", "testtype", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "2", Rating.IRRELEVANT.ordinal())); + rated.add(new RatedDocument("test_other", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test_other", "1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "2", Rating.IRRELEVANT.ordinal())); // the following search hits contain only the last three documents - EvalQueryQuality evaluated = (new Precision()).evaluate("id", - toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated.subList(2, 5), "test"), rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, - ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); - } - - public void testPrecisionAtFiveCorrectType() { - List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "other_type", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "other_type", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "2", Rating.IRRELEVANT.ordinal())); - EvalQueryQuality evaluated = (new Precision()).evaluate("id", - toSearchHits(rated.subList(2, 5), "test", "testtype"), rated); - assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, - ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testIgnoreUnlabeled() { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "1", Rating.RELEVANT.ordinal())); // add an unlabeled search hit - SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test", "testtype"), 3); + SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test"), 3); searchHits[2] = new SearchHit(2, "2", new Text("testtype"), Collections.emptyMap()); searchHits[2].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); EvalQueryQuality evaluated = (new Precision()).evaluate("id", searchHits, rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, - ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); // also try with setting `ignore_unlabeled` @@ -139,8 +115,7 @@ public class PrecisionTests extends ESTestCase { prec.setIgnoreUnlabeled(true); evaluated = prec.evaluate("id", searchHits, rated); assertEquals((double) 2 / 2, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, - ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } @@ -150,11 +125,9 @@ public class PrecisionTests extends ESTestCase { hits[i] = new SearchHit(i, i + "", new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); } - EvalQueryQuality evaluated = (new Precision()).evaluate("id", hits, - Collections.emptyList()); + EvalQueryQuality evaluated = (new Precision()).evaluate("id", hits, Collections.emptyList()); assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); - assertEquals(0, - ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); // also try with setting `ignore_unlabeled` @@ -162,8 +135,7 @@ public class PrecisionTests extends ESTestCase { prec.setIgnoreUnlabeled(true); evaluated = prec.evaluate("id", hits, Collections.emptyList()); assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); - assertEquals(0, - ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } @@ -201,8 +173,7 @@ public class PrecisionTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { Precision testItem = createTestItem(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - XContentBuilder shuffled = shuffleXContent( - testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); itemParser.nextToken(); @@ -223,8 +194,7 @@ public class PrecisionTests extends ESTestCase { public void testEqualsAndHash() throws IOException { Precision testItem = createTestItem(); - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - RankEvalTestHelper.copy(testItem, Precision::new)); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), RankEvalTestHelper.copy(testItem, Precision::new)); } private static Precision mutateTestItem(Precision original) { @@ -236,16 +206,15 @@ public class PrecisionTests extends ESTestCase { List mutators = new ArrayList<>(); mutators.add(() -> precision.setIgnoreUnlabeled(!ignoreUnlabeled)); - mutators.add(() -> precision.setRelevantRatingThreshhold( - randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10)))); + mutators.add(() -> precision.setRelevantRatingThreshhold(randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10)))); randomFrom(mutators).run(); return precision; } - private static SearchHit[] toSearchHits(List rated, String index, String type) { + private static SearchHit[] toSearchHits(List rated, String index) { SearchHit[] hits = new SearchHit[rated.size()]; for (int i = 0; i < rated.size(); i++) { - hits[i] = new SearchHit(i, i + "", new Text(type), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index fdae133afc2..22389ee7eec 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -65,7 +65,6 @@ public class RankEvalRequestIT extends ESIntegTestCase { public void testPrecisionAtRequest() { List indices = Arrays.asList(new String[] { "test" }); - List types = Arrays.asList(new String[] { "testtype" }); List specifications = new ArrayList<>(); SearchSourceBuilder testQuery = new SearchSourceBuilder(); @@ -73,14 +72,12 @@ public class RankEvalRequestIT extends ESIntegTestCase { RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", createRelevant("2", "3", "4", "5"), testQuery); amsterdamRequest.setIndices(indices); - amsterdamRequest.setTypes(types); amsterdamRequest.setSummaryFields(Arrays.asList(new String[] { "text", "title" })); specifications.add(amsterdamRequest); RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("1"), testQuery); berlinRequest.setIndices(indices); - berlinRequest.setTypes(types); berlinRequest.setSummaryFields(Arrays.asList(new String[] { "text", "title" })); specifications.add(berlinRequest); @@ -135,7 +132,6 @@ public class RankEvalRequestIT extends ESIntegTestCase { */ public void testBadQuery() { List indices = Arrays.asList(new String[] { "test" }); - List types = Arrays.asList(new String[] { "testtype" }); List specifications = new ArrayList<>(); SearchSourceBuilder amsterdamQuery = new SearchSourceBuilder(); @@ -143,7 +139,6 @@ public class RankEvalRequestIT extends ESIntegTestCase { RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", createRelevant("2", "3", "4", "5"), amsterdamQuery); amsterdamRequest.setIndices(indices); - amsterdamRequest.setTypes(types); specifications.add(amsterdamRequest); SearchSourceBuilder brokenQuery = new SearchSourceBuilder(); @@ -151,7 +146,6 @@ public class RankEvalRequestIT extends ESIntegTestCase { RatedRequest brokenRequest = new RatedRequest("broken_query", createRelevant("1"), brokenQuery); brokenRequest.setIndices(indices); - brokenRequest.setTypes(types); specifications.add(brokenRequest); RankEvalSpec task = new RankEvalSpec(specifications, new Precision()); @@ -173,7 +167,7 @@ public class RankEvalRequestIT extends ESIntegTestCase { private static List createRelevant(String... docs) { List relevant = new ArrayList<>(); for (String doc : docs) { - relevant.add(new RatedDocument("test", "testtype", doc, Rating.RELEVANT.ordinal())); + relevant.add(new RatedDocument("test", doc, Rating.RELEVANT.ordinal())); } return relevant; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 75e72be2e12..8a365672a24 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -78,18 +78,15 @@ public class RankEvalSpecTests extends ESTestCase { } templates = new HashSet<>(); - templates.add(new ScriptWithId("templateId", - new Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, script, params))); + templates.add(new ScriptWithId("templateId", new Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, script, params))); Map templateParams = new HashMap<>(); templateParams.put("key", "value"); - RatedRequest ratedRequest = new RatedRequest("id", - Arrays.asList(RatedDocumentTests.createRatedDocument()), templateParams, + RatedRequest ratedRequest = new RatedRequest("id", Arrays.asList(RatedDocumentTests.createRatedDocument()), templateParams, "templateId"); ratedRequests = Arrays.asList(ratedRequest); } else { - RatedRequest ratedRequest = new RatedRequest("id", - Arrays.asList(RatedDocumentTests.createRatedDocument()), + RatedRequest ratedRequest = new RatedRequest("id", Arrays.asList(RatedDocumentTests.createRatedDocument()), new SearchSourceBuilder()); ratedRequests = Arrays.asList(ratedRequest); } @@ -101,15 +98,11 @@ public class RankEvalSpecTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { RankEvalSpec testItem = createTestItem(); - XContentBuilder shuffled = shuffleXContent( - testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); try (XContentParser parser = createParser(JsonXContent.jsonXContent, shuffled.bytes())) { RankEvalSpec parsedItem = RankEvalSpec.parse(parser); // IRL these come from URL parameters - see RestRankEvalAction - // TODO Do we still need this? - // parsedItem.getRatedRequests().stream().forEach(e -> - // {e.setIndices(indices); e.setTypes(types);}); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); @@ -120,17 +113,13 @@ public class RankEvalSpecTests extends ESTestCase { RankEvalSpec original = createTestItem(); List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, - MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, - Precision.NAME, Precision::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, - DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, - ReciprocalRank.NAME, ReciprocalRank::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, + DiscountedCumulativeGain::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); - RankEvalSpec deserialized = RankEvalTestHelper.copy(original, RankEvalSpec::new, - new NamedWriteableRegistry(namedWriteables)); + RankEvalSpec deserialized = RankEvalTestHelper.copy(original, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables)); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); @@ -140,20 +129,15 @@ public class RankEvalSpecTests extends ESTestCase { RankEvalSpec testItem = createTestItem(); List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, - MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, - Precision.NAME, Precision::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, - DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, - ReciprocalRank.NAME, ReciprocalRank::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, + DiscountedCumulativeGain::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); - RankEvalSpec mutant = RankEvalTestHelper.copy(testItem, RankEvalSpec::new, - new NamedWriteableRegistry(namedWriteables)); + RankEvalSpec mutant = RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables)); RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(mutant), - RankEvalTestHelper.copy(testItem, RankEvalSpec::new, - new NamedWriteableRegistry(namedWriteables))); + RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables))); } private static RankEvalSpec mutateTestItem(RankEvalSpec mutant) { @@ -164,8 +148,7 @@ public class RankEvalSpecTests extends ESTestCase { int mutate = randomIntBetween(0, 2); switch (mutate) { case 0: - RatedRequest request = RatedRequestsTests.createTestItem(new ArrayList<>(), - new ArrayList<>(), true); + RatedRequest request = RatedRequestsTests.createTestItem(new ArrayList<>(), true); ratedRequests.add(request); break; case 1: @@ -176,16 +159,7 @@ public class RankEvalSpecTests extends ESTestCase { } break; case 2: - if (templates.size() > 0) { - String mutatedTemplate = randomAlphaOfLength(10); - templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, - new HashMap<>())); - } else { - String mutatedTemplate = randomValueOtherThanMany(templates::containsValue, - () -> randomAlphaOfLength(10)); - templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate, - new HashMap<>())); - } + templates.put("mutation", new Script(ScriptType.INLINE, "mustache", randomAlphaOfLength(10), new HashMap<>())); break; default: throw new IllegalStateException("Requested to modify more than available parameters."); @@ -202,28 +176,24 @@ public class RankEvalSpecTests extends ESTestCase { public void testMissingRatedRequestsFailsParsing() { RankedListQualityMetric metric = new Precision(); - expectThrows(IllegalStateException.class, - () -> new RankEvalSpec(new ArrayList<>(), metric)); + expectThrows(IllegalStateException.class, () -> new RankEvalSpec(new ArrayList<>(), metric)); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(null, metric)); } public void testMissingMetricFailsParsing() { List strings = Arrays.asList("value"); - List ratedRequests = randomList( - () -> RatedRequestsTests.createTestItem(strings, strings, randomBoolean())); + List ratedRequests = randomList(() -> RatedRequestsTests.createTestItem(strings, randomBoolean())); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, null)); } public void testMissingTemplateAndSearchRequestFailsParsing() { - List ratedDocs = Arrays - .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); Map params = new HashMap<>(); params.put("key", "value"); RatedRequest request = new RatedRequest("id", ratedDocs, params, "templateId"); List ratedRequests = Arrays.asList(request); - expectThrows(IllegalStateException.class, - () -> new RankEvalSpec(ratedRequests, new Precision())); + expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, new Precision())); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java index 15ae5d03537..088db7df814 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java @@ -36,6 +36,7 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +// TODO replace by infra from ESTestCase public class RankEvalTestHelper { public static void testHashCodeAndEquals(T testItem, T mutation, T secondCopy) { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index 9e0d16620d0..672d464a386 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -32,18 +32,16 @@ public class RatedDocumentTests extends ESTestCase { public static RatedDocument createRatedDocument() { String index = randomAlphaOfLength(10); - String type = randomAlphaOfLength(10); String docId = randomAlphaOfLength(10); int rating = randomInt(); - return new RatedDocument(index, type, docId, rating); + return new RatedDocument(index, docId, rating); } public void testXContentParsing() throws IOException { RatedDocument testItem = createRatedDocument(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - XContentBuilder shuffled = shuffleXContent( - testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { RatedDocument parsedItem = RatedDocument.fromXContent(itemParser); assertNotSame(testItem, parsedItem); @@ -62,29 +60,22 @@ public class RatedDocumentTests extends ESTestCase { public void testEqualsAndHash() throws IOException { RatedDocument testItem = createRatedDocument(); - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - RankEvalTestHelper.copy(testItem, RatedDocument::new)); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), RankEvalTestHelper.copy(testItem, RatedDocument::new)); } public void testInvalidParsing() { - expectThrows(IllegalArgumentException.class, - () -> new RatedDocument(null, "abc", "abc", 10)); - expectThrows(IllegalArgumentException.class, () -> new RatedDocument("", "abc", "abc", 10)); - expectThrows(IllegalArgumentException.class, - () -> new RatedDocument("abc", null, "abc", 10)); - expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "", "abc", 10)); - expectThrows(IllegalArgumentException.class, - () -> new RatedDocument("abc", "abc", null, 10)); - expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "abc", "", 10)); + expectThrows(IllegalArgumentException.class, () -> new RatedDocument(null, "abc", 10)); + expectThrows(IllegalArgumentException.class, () -> new RatedDocument("", "abc", 10)); + expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "", 10)); + expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", null, 10)); } private static RatedDocument mutateTestItem(RatedDocument original) { int rating = original.getRating(); String index = original.getIndex(); - String type = original.getType(); String docId = original.getDocID(); - switch (randomIntBetween(0, 3)) { + switch (randomIntBetween(0, 2)) { case 0: rating = randomValueOtherThan(rating, () -> randomInt()); break; @@ -92,14 +83,11 @@ public class RatedDocumentTests extends ESTestCase { index = randomValueOtherThan(index, () -> randomAlphaOfLength(10)); break; case 2: - type = randomValueOtherThan(type, () -> randomAlphaOfLength(10)); - break; - case 3: docId = randomValueOtherThan(docId, () -> randomAlphaOfLength(10)); break; default: throw new IllegalStateException("The test should only allow two parameters mutated"); } - return new RatedDocument(index, type, docId, rating); + return new RatedDocument(index, docId, rating); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 3b7d64ff4c7..58a9f998183 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -57,9 +57,9 @@ public class RatedRequestsTests extends ESTestCase { */ @BeforeClass public static void init() { - xContentRegistry = new NamedXContentRegistry(Stream.of( - new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents().stream()) - .flatMap(Function.identity()).collect(toList())); + xContentRegistry = new NamedXContentRegistry( + Stream.of(new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents().stream()).flatMap(Function.identity()) + .collect(toList())); } @AfterClass @@ -72,8 +72,7 @@ public class RatedRequestsTests extends ESTestCase { return xContentRegistry; } - public static RatedRequest createTestItem(List indices, List types, - boolean forceRequest) { + public static RatedRequest createTestItem(List indices, boolean forceRequest) { String requestId = randomAlphaOfLength(50); List ratedDocs = new ArrayList<>(); @@ -105,12 +104,10 @@ public class RatedRequestsTests extends ESTestCase { if (params.size() == 0) { ratedRequest = new RatedRequest(requestId, ratedDocs, testRequest); ratedRequest.setIndices(indices); - ratedRequest.setTypes(types); ratedRequest.setSummaryFields(summaryFields); } else { ratedRequest = new RatedRequest(requestId, ratedDocs, params, randomAlphaOfLength(5)); ratedRequest.setIndices(indices); - ratedRequest.setTypes(types); ratedRequest.setSummaryFields(summaryFields); } return ratedRequest; @@ -123,25 +120,14 @@ public class RatedRequestsTests extends ESTestCase { indices.add(randomAlphaOfLengthBetween(0, 50)); } - List types = new ArrayList<>(); - size = randomIntBetween(0, 20); - for (int i = 0; i < size; i++) { - types.add(randomAlphaOfLengthBetween(0, 50)); - } - - RatedRequest testItem = createTestItem(indices, types, randomBoolean()); + RatedRequest testItem = createTestItem(indices, randomBoolean()); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - XContentBuilder shuffled = shuffleXContent( - testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); RatedRequest parsedItem = RatedRequest.fromXContent(itemParser); - parsedItem.setIndices(indices); // IRL these come from URL - // parameters - see - // RestRankEvalAction - parsedItem.setTypes(types); // IRL these come from URL parameters - - // see RestRankEvalAction + parsedItem.setIndices(indices); // IRL these come from URL parameters - see RestRankEvalAction assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); @@ -155,20 +141,12 @@ public class RatedRequestsTests extends ESTestCase { indices.add(randomAlphaOfLengthBetween(0, 50)); } - List types = new ArrayList<>(); - size = randomIntBetween(0, 20); - for (int i = 0; i < size; i++) { - types.add(randomAlphaOfLengthBetween(0, 50)); - } - - RatedRequest original = createTestItem(indices, types, randomBoolean()); + RatedRequest original = createTestItem(indices, randomBoolean()); List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, - MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - RatedRequest deserialized = RankEvalTestHelper.copy(original, RatedRequest::new, - new NamedWriteableRegistry(namedWriteables)); + RatedRequest deserialized = RankEvalTestHelper.copy(original, RatedRequest::new, new NamedWriteableRegistry(namedWriteables)); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); @@ -181,21 +159,13 @@ public class RatedRequestsTests extends ESTestCase { indices.add(randomAlphaOfLengthBetween(0, 50)); } - List types = new ArrayList<>(); - size = randomIntBetween(0, 20); - for (int i = 0; i < size; i++) { - types.add(randomAlphaOfLengthBetween(0, 50)); - } - - RatedRequest testItem = createTestItem(indices, types, randomBoolean()); + RatedRequest testItem = createTestItem(indices, randomBoolean()); List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, - MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - RankEvalTestHelper.copy(testItem, RatedRequest::new, - new NamedWriteableRegistry(namedWriteables))); + RankEvalTestHelper.copy(testItem, RatedRequest::new, new NamedWriteableRegistry(namedWriteables))); } private static RatedRequest mutateTestItem(RatedRequest original) { @@ -203,12 +173,11 @@ public class RatedRequestsTests extends ESTestCase { SearchSourceBuilder testRequest = original.getTestRequest(); List ratedDocs = original.getRatedDocs(); List indices = original.getIndices(); - List types = original.getTypes(); Map params = original.getParams(); List summaryFields = original.getSummaryFields(); String templateId = original.getTemplateId(); - int mutate = randomIntBetween(0, 5); + int mutate = randomIntBetween(0, 4); switch (mutate) { case 0: id = randomValueOtherThan(id, () -> randomAlphaOfLength(10)); @@ -231,120 +200,95 @@ public class RatedRequestsTests extends ESTestCase { } break; case 2: - ratedDocs = Arrays.asList(randomValueOtherThanMany(ratedDocs::contains, - () -> RatedDocumentTests.createRatedDocument())); + ratedDocs = Arrays.asList(randomValueOtherThanMany(ratedDocs::contains, () -> RatedDocumentTests.createRatedDocument())); break; case 3: - indices = Arrays.asList( - randomValueOtherThanMany(indices::contains, () -> randomAlphaOfLength(10))); + indices = Arrays.asList(randomValueOtherThanMany(indices::contains, () -> randomAlphaOfLength(10))); break; case 4: - types = Arrays.asList( - randomValueOtherThanMany(types::contains, () -> randomAlphaOfLength(10))); - break; - case 5: - summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains, - () -> randomAlphaOfLength(10))); + summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains, () -> randomAlphaOfLength(10))); break; default: throw new IllegalStateException("Requested to modify more than available parameters."); } - RatedRequest ratedRequest = new RatedRequest(id, ratedDocs, testRequest, params, - templateId); + RatedRequest ratedRequest = new RatedRequest(id, ratedDocs, testRequest, params, templateId); ratedRequest.setIndices(indices); - ratedRequest.setTypes(types); ratedRequest.setSummaryFields(summaryFields); return ratedRequest; } public void testDuplicateRatedDocThrowsException() { - List ratedDocs = Arrays.asList( - new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1), - new RatedDocument(new DocumentKey("index1", "type1", "id1"), 5)); + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1), + new RatedDocument(new DocumentKey("index1", "id1"), 5)); // search request set, no summary fields IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder())); - assertEquals( - "Found duplicate rated document key [{\"_index\":\"index1\"," - + "\"_type\":\"type1\",\"_id\":\"id1\"}]", + assertEquals("Found duplicate rated document key [{\"_index\":\"index1\",\"_id\":\"id1\"}]", ex.getMessage()); // templated path, no summary fields Map params = new HashMap<>(); params.put("key", "value"); - ex = expectThrows(IllegalArgumentException.class, - () -> new RatedRequest("id", ratedDocs, params, "templateId")); - assertEquals( - "Found duplicate rated document key [{\"_index\":\"index1\"," - + "\"_type\":\"type1\",\"_id\":\"id1\"}]", + ex = expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, params, "templateId")); + assertEquals("Found duplicate rated document key [{\"_index\":\"index1\",\"_id\":\"id1\"}]", ex.getMessage()); } public void testNullSummaryFieldsTreatment() { - List ratedDocs = Arrays - .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder()); expectThrows(IllegalArgumentException.class, () -> request.setSummaryFields(null)); } public void testNullParamsTreatment() { - List ratedDocs = Arrays - .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); - RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, - null); + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, null); assertNotNull(request.getParams()); } public void testSettingParamsAndRequestThrows() { - List ratedDocs = Arrays - .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); Map params = new HashMap<>(); params.put("key", "value"); - expectThrows(IllegalArgumentException.class, - () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), params, null)); + expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), params, null)); } public void testSettingNeitherParamsNorRequestThrows() { - List ratedDocs = Arrays - .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); - expectThrows(IllegalArgumentException.class, - () -> new RatedRequest("id", ratedDocs, null, null)); - expectThrows(IllegalArgumentException.class, - () -> new RatedRequest("id", ratedDocs, null, new HashMap<>(), "templateId")); + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null)); + expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, new HashMap<>(), "templateId")); } public void testSettingParamsWithoutTemplateIdThrows() { - List ratedDocs = Arrays - .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); Map params = new HashMap<>(); params.put("key", "value"); - expectThrows(IllegalArgumentException.class, - () -> new RatedRequest("id", ratedDocs, null, params, null)); + expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, params, null)); } public void testSettingTemplateIdAndRequestThrows() { - List ratedDocs = Arrays - .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); - expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, - new SearchSourceBuilder(), null, "templateId")); + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + expectThrows(IllegalArgumentException.class, + () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, "templateId")); } public void testSettingTemplateIdNoParamsThrows() { - List ratedDocs = Arrays - .asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1)); - expectThrows(IllegalArgumentException.class, - () -> new RatedRequest("id", ratedDocs, null, null, "templateId")); + List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null, "templateId")); } /** - * test that modifying the order of index/type/docId to make sure it doesn't + * test that modifying the order of index/docId to make sure it doesn't * matter for parsing xContent */ public void testParseFromXContent() throws IOException { - String querySpecString = " {\n" + " \"id\": \"my_qa_query\",\n" + " \"request\": {\n" - + " \"query\": {\n" + " \"bool\": {\n" + String querySpecString = " {\n" + + " \"id\": \"my_qa_query\",\n" + + " \"request\": {\n" + + " \"query\": {\n" + + " \"bool\": {\n" + " \"must\": [\n" + " {\"match\": {\"beverage\": \"coffee\"}},\n" + " {\"term\": {\"browser\": {\"value\": \"safari\"}}},\n" @@ -352,16 +296,15 @@ public class RatedRequestsTests extends ESTestCase { + " {\"value\": \"morning\",\"boost\": 2}}},\n" + " {\"term\": {\"ip_location\": " + " {\"value\": \"ams\",\"boost\": 10}}}]}\n" - + " },\n" + " \"size\": 10\n" + " },\n" + + " },\n" + + " \"size\": 10\n" + + " },\n" + " \"summary_fields\" : [\"title\"],\n" + " \"ratings\": [\n" - + " {\"_index\": \"test\", \"_type\": \"testtype\", " - + " \"_id\": \"1\", \"rating\" : 1 }, " - + " {\"_type\": \"testtype\", \"_index\": \"test\", " - + " \"_id\": \"2\", \"rating\" : 0 }, " - + " {\"_id\": \"3\", \"_index\": \"test\", " - + " \"_type\": \"testtype\", \"rating\" : 1 }]\n" - + "}"; + + " {\"_index\": \"test\" , \"_id\": \"1\", \"rating\" : 1 },\n" + + " {\"_index\": \"test\", \"rating\" : 0, \"_id\": \"2\"},\n" + + " {\"_id\": \"3\", \"_index\": \"test\", \"rating\" : 1} ]" + + "}\n"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, querySpecString)) { RatedRequest specification = RatedRequest.fromXContent(parser); assertEquals("my_qa_query", specification.getId()); @@ -371,7 +314,6 @@ public class RatedRequestsTests extends ESTestCase { for (int i = 0; i < 3; i++) { assertEquals("" + (i + 1), ratedDocs.get(i).getDocID()); assertEquals("test", ratedDocs.get(i).getIndex()); - assertEquals("testtype", ratedDocs.get(i).getType()); if (i == 1) { assertEquals(0, ratedDocs.get(i).getRating()); } else { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 4d0f3347acd..273a2ff7a5c 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -45,75 +45,67 @@ public class ReciprocalRankTests extends ESTestCase { int searchHits = randomIntBetween(1, 50); - SearchHit[] hits = createSearchHits(0, searchHits, "test", "type"); + SearchHit[] hits = createSearchHits(0, searchHits, "test"); List ratedDocs = new ArrayList<>(); int relevantAt = randomIntBetween(0, searchHits); for (int i = 0; i <= searchHits; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), - Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), - Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.IRRELEVANT.ordinal())); } } int rankAtFirstRelevant = relevantAt + 1; EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE); - assertEquals(rankAtFirstRelevant, - ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); + assertEquals(rankAtFirstRelevant, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); // check that if we have fewer search hits than relevant doc position, // we don't find any result and get 0.0 quality level reciprocalRank = new ReciprocalRank(); - evaluation = reciprocalRank.evaluate("id", Arrays.copyOfRange(hits, 0, relevantAt), - ratedDocs); + evaluation = reciprocalRank.evaluate("id", Arrays.copyOfRange(hits, 0, relevantAt), ratedDocs); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); } public void testEvaluationOneRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); - SearchHit[] hits = createSearchHits(0, 9, "test", "type"); + SearchHit[] hits = createSearchHits(0, 9, "test"); List ratedDocs = new ArrayList<>(); // mark one of the ten docs relevant int relevantAt = randomIntBetween(0, 9); for (int i = 0; i <= 20; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), - Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), - Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.IRRELEVANT.ordinal())); } } EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE); - assertEquals(relevantAt + 1, - ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); + assertEquals(relevantAt + 1, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } /** - * test that the relevant rating threshold can be set to something larger - * than 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant, - * so first relevant doc has third ranking position, so RR should be 1/3 + * test that the relevant rating threshold can be set to something larger than + * 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant, so first + * relevant doc has third ranking position, so RR should be 1/3 */ public void testPrecisionAtFiveRelevanceThreshold() { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "testtype", "0", 0)); - rated.add(new RatedDocument("test", "testtype", "1", 1)); - rated.add(new RatedDocument("test", "testtype", "2", 2)); - rated.add(new RatedDocument("test", "testtype", "3", 3)); - rated.add(new RatedDocument("test", "testtype", "4", 4)); - SearchHit[] hits = createSearchHits(0, 5, "test", "testtype"); + rated.add(new RatedDocument("test", "0", 0)); + rated.add(new RatedDocument("test", "1", 1)); + rated.add(new RatedDocument("test", "2", 2)); + rated.add(new RatedDocument("test", "3", 3)); + rated.add(new RatedDocument("test", "4", 4)); + SearchHit[] hits = createSearchHits(0, 5, "test"); ReciprocalRank reciprocalRank = new ReciprocalRank(); reciprocalRank.setRelevantRatingThreshhold(2); EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, rated); assertEquals((double) 1 / 3, evaluation.getQualityLevel(), 0.00001); - assertEquals(3, - ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); + assertEquals(3, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } public void testCombine() { @@ -127,7 +119,7 @@ public class ReciprocalRankTests extends ESTestCase { public void testEvaluationNoRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); - SearchHit[] hits = createSearchHits(0, 9, "test", "type"); + SearchHit[] hits = createSearchHits(0, 9, "test"); List ratedDocs = new ArrayList<>(); EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); @@ -136,8 +128,7 @@ public class ReciprocalRankTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { ReciprocalRank testItem = createTestItem(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - XContentBuilder shuffled = shuffleXContent( - testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); itemParser.nextToken(); @@ -149,13 +140,13 @@ public class ReciprocalRankTests extends ESTestCase { } /** - * Create SearchHits for testing, starting from dociId 'from' up to docId - * 'to'. The search hits index and type also need to be provided + * Create SearchHits for testing, starting from dociId 'from' up to docId 'to'. + * The search hits index also need to be provided */ - private static SearchHit[] createSearchHits(int from, int to, String index, String type) { + private static SearchHit[] createSearchHits(int from, int to, String index) { SearchHit[] hits = new SearchHit[to + 1 - from]; for (int i = from; i <= to; i++) { - hits[i] = new SearchHit(i, i + "", new Text(type), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null)); } return hits; @@ -185,8 +176,7 @@ public class ReciprocalRankTests extends ESTestCase { private static ReciprocalRank mutateTestItem(ReciprocalRank testItem) { int relevantThreshold = testItem.getRelevantRatingThreshold(); ReciprocalRank rank = new ReciprocalRank(); - rank.setRelevantRatingThreshhold( - randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10))); + rank.setRelevantRatingThreshhold(randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10))); return rank; } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml index 9c683bc474c..991b4c84dc0 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml @@ -46,14 +46,14 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, "ratings": [ - {"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 0}, - {"_index": "foo", "_type": "bar", "_id": "doc2", "rating": 1}, - {"_index": "foo", "_type": "bar", "_id": "doc3", "rating": 1}] + {"_index": "foo", "_id": "doc1", "rating": 0}, + {"_index": "foo", "_id": "doc2", "rating": 1}, + {"_index": "foo", "_id": "doc3", "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, - "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] + "ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}] } ], "metric" : { "precision": { "ignore_unlabeled" : true }} @@ -61,7 +61,7 @@ - match: { rank_eval.quality_level: 1} - match: { rank_eval.details.amsterdam_query.quality_level: 1.0} - - match: { rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} + - match: { rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_id": "doc4"}]} - match: { rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}} - length: { rank_eval.details.amsterdam_query.hits: 3} @@ -73,7 +73,7 @@ - is_false: rank_eval.details.amsterdam_query.hits.2.rating - match: { rank_eval.details.berlin_query.quality_level: 1.0} - - match: { rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} + - match: { rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_id": "doc4"}]} - match: { rank_eval.details.berlin_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}} - length: { rank_eval.details.berlin_query.hits: 2} - match: { rank_eval.details.berlin_query.hits.0.hit._id: "doc1" } @@ -130,13 +130,13 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, # doc4 should be returned in third position, so reciprocal rank is 1/3 - "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc4", "rating": 1}] + "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 - "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc4", "rating": 1}] + "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}] } ], "metric" : { "reciprocal_rank": {} } @@ -146,9 +146,9 @@ - match: {rank_eval.quality_level: 0.41666666666666663} - match: {rank_eval.details.amsterdam_query.quality_level: 0.3333333333333333} - match: {rank_eval.details.amsterdam_query.metric_details: {"first_relevant": 3}} - - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc2"}, - {"_index": "foo", "_type": "bar", "_id": "doc3"} ]} + - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_id": "doc2"}, + {"_index": "foo", "_id": "doc3"} ]} - match: {rank_eval.details.berlin_query.quality_level: 0.5} - match: {rank_eval.details.berlin_query.metric_details: {"first_relevant": 2}} - - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc1"}]} + - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_id": "doc1"}]} diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml index 580f21c1441..c6c1c87e753 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml @@ -54,12 +54,12 @@ "id": "dcg_query", "request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] }, "ratings": [ - {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] + {"_index" : "foo", "_id" : "doc1", "rating": 3}, + {"_index" : "foo", "_id" : "doc2", "rating": 2}, + {"_index" : "foo", "_id" : "doc3", "rating": 3}, + {"_index" : "foo", "_id" : "doc4", "rating": 0}, + {"_index" : "foo", "_id" : "doc5", "rating": 1}, + {"_index" : "foo", "_id" : "doc6", "rating": 2}] } ], "metric" : { "dcg": {}} @@ -79,12 +79,12 @@ "id": "dcg_query_reverse", "request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] }, "ratings": [ - {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] + {"_index" : "foo", "_id" : "doc1", "rating": 3}, + {"_index" : "foo", "_id" : "doc2", "rating": 2}, + {"_index" : "foo", "_id" : "doc3", "rating": 3}, + {"_index" : "foo", "_id" : "doc4", "rating": 0}, + {"_index" : "foo", "_id" : "doc5", "rating": 1}, + {"_index" : "foo", "_id" : "doc6", "rating": 2}] }, ], "metric" : { "dcg": { }} @@ -104,23 +104,23 @@ "id": "dcg_query", "request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] }, "ratings": [ - {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] + {"_index" : "foo", "_id" : "doc1", "rating": 3}, + {"_index" : "foo", "_id" : "doc2", "rating": 2}, + {"_index" : "foo", "_id" : "doc3", "rating": 3}, + {"_index" : "foo", "_id" : "doc4", "rating": 0}, + {"_index" : "foo", "_id" : "doc5", "rating": 1}, + {"_index" : "foo", "_id" : "doc6", "rating": 2}] }, { "id": "dcg_query_reverse", "request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] }, "ratings": [ - {"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1}, - {"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}] + {"_index" : "foo", "_id" : "doc1", "rating": 3}, + {"_index" : "foo", "_id" : "doc2", "rating": 2}, + {"_index" : "foo", "_id" : "doc3", "rating": 3}, + {"_index" : "foo", "_id" : "doc4", "rating": 0}, + {"_index" : "foo", "_id" : "doc5", "rating": 1}, + {"_index" : "foo", "_id" : "doc6", "rating": 2}] }, ], "metric" : { "dcg": { }} diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml index f3b7bf4627e..130da28f3b1 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml @@ -19,12 +19,12 @@ "id": "amsterdam_query", "request": { "query": { "match_all" : { }}}, "ratings": [ - {"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] + {"_index": "foo", "_id": "doc1", "rating": 1}] }, { "id" : "invalid_query", "request": { "query": { "range" : { "bar" : { "from" : "Basel", "time_zone": "+01:00" }}}}, - "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] + "ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}] } ], "metric" : { "precision": { "ignore_unlabeled" : true }} diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java index 09da73a1f7f..d4a0411349d 100644 --- a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java @@ -26,9 +26,9 @@ import org.elasticsearch.index.rankeval.RankEvalRequest; import org.elasticsearch.index.rankeval.RankEvalRequestBuilder; import org.elasticsearch.index.rankeval.RankEvalResponse; import org.elasticsearch.index.rankeval.RankEvalSpec; +import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId; import org.elasticsearch.index.rankeval.RatedDocument; import org.elasticsearch.index.rankeval.RatedRequest; -import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; @@ -82,7 +82,6 @@ public class SmokeMultipleTemplatesIT extends ESIntegTestCase { public void testPrecisionAtRequest() throws IOException { List indices = Arrays.asList(new String[] { "test" }); - List types = Arrays.asList(new String[] { "testtype" }); List specifications = new ArrayList<>(); Map ams_params = new HashMap<>(); @@ -90,7 +89,6 @@ public class SmokeMultipleTemplatesIT extends ESIntegTestCase { RatedRequest amsterdamRequest = new RatedRequest( "amsterdam_query", createRelevant("2", "3", "4", "5"), ams_params, MATCH_TEMPLATE); amsterdamRequest.setIndices(indices); - amsterdamRequest.setTypes(types); specifications.add(amsterdamRequest); @@ -99,12 +97,11 @@ public class SmokeMultipleTemplatesIT extends ESIntegTestCase { RatedRequest berlinRequest = new RatedRequest( "berlin_query", createRelevant("1"), berlin_params, MATCH_TEMPLATE); berlinRequest.setIndices(indices); - berlinRequest.setTypes(types); specifications.add(berlinRequest); Precision metric = new Precision(); - ScriptWithId template = + ScriptWithId template = new ScriptWithId( MATCH_TEMPLATE, new Script( @@ -124,7 +121,7 @@ public class SmokeMultipleTemplatesIT extends ESIntegTestCase { private static List createRelevant(String... docs) { List relevant = new ArrayList<>(); for (String doc : docs) { - relevant.add(new RatedDocument("test", "testtype", doc, Rating.RELEVANT.ordinal())); + relevant.add(new RatedDocument("test", doc, Rating.RELEVANT.ordinal())); } return relevant; } diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yml b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yml index 40db3724a9a..f6e77115a23 100644 --- a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yml +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yml @@ -41,22 +41,22 @@ - do: rank_eval: body: { - "templates": [ { "id": "match", "template": {"inline": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" }} ], + "templates": [ { "id": "match", "template": {"source": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" }} ], "requests" : [ { "id": "amsterdam_query", "params": { "query_string": "amsterdam" }, "template_id": "match", "ratings": [ - {"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 0}, - {"_index": "foo", "_type": "bar", "_id": "doc2", "rating": 1}, - {"_index": "foo", "_type": "bar", "_id": "doc3", "rating": 1}] + {"_index": "foo", "_id": "doc1", "rating": 0}, + {"_index": "foo", "_id": "doc2", "rating": 1}, + {"_index": "foo", "_id": "doc3", "rating": 1}] }, { "id" : "berlin_query", "params": { "query_string": "berlin" }, "template_id": "match", - "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] + "ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}] } ], "metric" : { "precision": { }} From d9e67a2c95b5cc28e91343e5f40794d83729118a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 9 Nov 2017 17:57:26 +0100 Subject: [PATCH 107/297] Extending `_rank_eval` documentation --- docs/reference/search/rank-eval.asciidoc | 295 +++++++++--------- ...rocalRank.java => MeanReciprocalRank.java} | 22 +- .../{Precision.java => PrecisionAtK.java} | 22 +- .../index/rankeval/RankEvalPlugin.java | 10 +- .../rankeval/RankedListQualityMetric.java | 8 +- .../index/rankeval/RestRankEvalAction.java | 164 +++------- .../index/rankeval/EvalQueryQualityTests.java | 4 +- .../index/rankeval/PrecisionTests.java | 74 ++--- .../index/rankeval/RankEvalRequestIT.java | 4 +- .../index/rankeval/RankEvalSpecTests.java | 18 +- .../index/rankeval/ReciprocalRankTests.java | 40 +-- .../rest-api-spec/test/rank_eval/10_basic.yml | 4 +- 12 files changed, 311 insertions(+), 354 deletions(-) rename modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/{ReciprocalRank.java => MeanReciprocalRank.java} (88%) rename modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/{Precision.java => PrecisionAtK.java} (91%) diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index ee21ca19aa9..321791e6b03 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -1,149 +1,150 @@ -[[rank-eval]] -= Ranking Evaluation +[[search-rank-eval]] +== Ranking Evaluation API -[partintro] --- +experimental[] -Imagine having built and deployed a search application: Users are happily -entering queries into your search frontend. Your application takes these -queries and creates a dedicated Elasticsearch query from that, and returns its -results back to the user. Imagine further that you are tasked with tweaking the -Elasticsearch query that is being created to return specific results for a -certain set of queries without breaking others. How should that be done? +The ranking evaluation API allows to evaluate the quality of ranked search +results over a set of typical search queries. Given this set of queries and a +list or manually rated documents, the `_rank_eval` endpoint calculates and +returns typical information retrieval metrics like _mean reciprocal rank_, +_precision_ or _discounted cumulative gain_. -One possible solution is to gather a sample of user queries representative of -how the search application is used, retrieve the search results that are being -returned. As a next step these search results would be manually annotated for -their relevancy to the original user query. Based on this set of rated requests -we can compute a couple of metrics telling us more about how many relevant -search results are being returned. +=== Overview -This is a nice approximation for how well our translation from user query to -Elasticsearch query works for providing the user with relevant search results. -Elasticsearch provides a ranking evaluation API that lets you compute scores for -your current ranking function based on annotated search results. --- +Search quality evaluation starts with looking at the users of your search application, and the things that they are searching for. +Users have a specific _information need_, e.g. they are looking for gift in a web shop or want to book a flight for their next holiday. +They usually enters some search terms into a search box or some other web form. +All of this information, together with meta information about the user (e.g. the browser, location, earlier preferences etc...) then gets translated into a query to the underlying search system. -== Plain ranking evaluation +The challenge for search engineers is to tweak this translation process from user entries to a concrete query in such a way, that the search results contain the most relevant information with respect to the users information_need. +This can only be done if the search result quality is evaluated constantly across a representative test suit of typical user queries, so that improvements in the rankings for one particular query doesn't negatively effect the ranking for other types of queries. -In its most simple form, for each query a set of ratings can be supplied: +In order to get started with search quality evaluation, three basic things a needed: + +. a collection of document you want to evaluate your query performance against, usually one or more indices +. a collection of typical search requests that users enter into your system +. a set of document ratings that judge the documents relevance with respect to a search request+ + It is important to note that one set of document ratings is needed per test query, and that + the relevance judgements are based on the _information_need_ of the user that entered the query. + +The ranking evaluation API provides a convenient way to use this information in a ranking evaluation request to calculate different search evaluation metrics. This gives a first estimation of your overall search quality and give you a measurement to optimize against when fine-tuning various aspect of the query generation in your application. + +== Ranking evaluation request structure + +In its most basic form, a request to the `_rank_eval` endpoint has two sections: [source,js] ----------------------------- -GET /twitter/tweet/_rank_eval +GET /my_index/_rank_eval { - "requests": [ - { - "id": "JFK query", <1> - "request": { - "query": { - "match": { - "title": { - "query": "JFK"}}}}, <2> - "ratings": [ <3> - { - "rating": 1.5, <4> - "_type": "tweet", <5> - "_id": "13736278", <6> - "_index": "twitter" <7> - }, - { - "rating": 1, - "_type": "tweet", - "_id": "30900421", - "_index": "twitter" - }], - "summary_fields": ["title"] <8> - }], - "metric": { <9> - "reciprocal_rank": {} - }, - "max_concurrent_searches": 10 <10> + "requests": [ ... ], <1> + "metric": { <2> + "reciprocal_rank": { ... } <3> + } } ------------------------------ -// CONSOLE -// TEST[setup:twitter] +// NOTCONSOLE -<1> A human readable id for the rated query (that will be re-used in the response to provide further details). -<2> The actual Elasticsearch query to execute. -<3> A set of ratings for how well a certain document fits as response for the query. -<4> A rating expressing how well the document fits the query, higher is better, are treated as int values. -<5> The type where the rated document lives. -<6> The id of the rated document. -<7> The index where the rated document lives. -<8> For a verbose response, specify which properties of a search hit should be returned in addition to index/type/id. -<9> A metric to use for evaluation. See below for a list. -<10> Maximum number of search requests to execute in parallel. Set to 10 by -default. +<1> a set of typical search requests to your system +<2> definition of the evaluation metric to calculate +<3> a specific metric and its parameters +The request section contains several search requests typical to your application, along with the document ratings for each particular search request, e.g. + +[source,js] +----------------------------- + "requests": [ + { + "id": "amsterdam_query", <1> + "request": { <2> + "query": { "match": { "text": "amsterdam" }} + }, + "ratings": [ <3> + { "_index": "my_index", "_id": "doc1", "rating": 0 }, + { "_index": "my_index", "_id": "doc2", "rating": 3}, + { "_index": "my_index", _id": "doc3", "rating": 1 } + ] + }, + { + "id": "berlin_query", + "request": { + "query": { "match": { "text": "berlin" }} + }, + "ratings": [ + { "_index": "my_index", "_id": "doc1", "rating": 1 } + ] + } + ] +------------------------------ +// NOTCONSOLE + +<1> the search requests id, used to group result details later +<2> the query that is being evaluated +<3> a list of document ratings, each entry containing the documents `_index` and `_id` together with +the rating of the documents relevance with regards to this search request + +A document `rating` can be any integer value that expresses the relevance of the document on a user defined scale. For some of the metrics, just giving a binary rating (e.g. `0` for irrelevant and `1` for relevant) will be sufficient, other metrics can use a more fine grained scale. == Template based ranking evaluation +As an alternative to having to provide a single query per test request, it is possible to specify query templates in the evaluation request and later refer to them. Queries with similar structure that only differ in their parameters don't have to be repeated all the time in the `requests` section this way. In typical search systems where user inputs usually get filled into a small set of query templates, this helps making the evaluation request more succinct. + [source,js] -------------------------------- -GET /twitter/tweet/_rank_eval -{ - "templates": [{ - "id": "match_query", - "template": { - "inline": { - "query": { - "match": { - "{{field}}": { - "query": "{{query_string}}"}}}}}}], <1> +GET /my_index/_rank_eval +{ + [...] + "templates": [ + { + "id": "match_one_field_query", <1> + "template": { <2> + "inline": { + "query": { + "match": { "{{field}}": { "query": "{{query_string}}" }} + } + } + } + } + ], "requests": [ - { - "id": "JFK query", - "ratings": [ - { - "rating": 1.5, - "_type": "tweet", - "_id": "13736278", - "_index": "twitter" - }, - { - "rating": 1, - "_type": "tweet", - "_id": "30900421", - "_index": "twitter" - }], - "params": { - "query_string": "JFK", <2> - "field": "opening_text" <2> - }, - "template_id": "match_query" - }], - "metric": { - "precision": { - "relevant_rating_threshold": 2 - } - } + { + "id": "amsterdam_query" + "ratings": [ ... ], + "template_id": "match_one_field_query", <3> + "params": { <4> + "query_string": "amsterdam", + "field": "text" + } + }, + [...] } -------------------------------- -// CONSOLE -// TEST[setup:twitter] +// NOTCONSOLE -<1> The template to use for every rated search request. -<2> The parameters to use to fill the template above. +<1> the template id +<2> the template definition to use +<3> a reference to a previously defined temlate +<4> the parameters to use to fill the template +== Available evaluation metrics -== Valid evaluation metrics +The `metric` section determines which of the available evaluation metrics is going to be used. +Currently, the following metrics are supported: -=== Precision +=== Precision at k (Prec@k) -Citing from https://en.wikipedia.org/wiki/Information_retrieval#Precision[Precision -page at Wikipedia]: -"Precision is the fraction of the documents retrieved that are relevant to the -user's information need." +This metric measures the number of relevant results in the top k search results. Its a form of the well known https://en.wikipedia.org/wiki/Information_retrieval#Precision[Precision] metric that only looks at the top k documents. It is the fraction of relevant documents in those first k +search. A precision at 10 (prec@10) value of 0.6 then means six out of the 10 top hits where +relevant with respect to the users information need. -Works well as an easy to explain evaluation metric. Caveat: All result positions -are treated equally. So a ranking of ten results that contains one relevant +This metric works well as a first and an easy to explain evaluation metric. +Documents in the collection need to be rated either as relevant or irrelevant with respect to the current query. Prec@k does not take into account where in the top k results the relevant documents occur, so a ranking of ten results that contains one relevant result in position 10 is equally good as a ranking of ten results that contains one relevant result in position 1. [source,js] -------------------------------- -GET /twitter/tweet/_rank_eval +GET /twitter/_rank_eval { "requests": [ { @@ -153,8 +154,8 @@ GET /twitter/tweet/_rank_eval }], "metric": { "precision": { - "relevant_rating_threshold": 1, <1> - "ignore_unlabeled": false <2> + "relevant_rating_threshold": 1, + "ignore_unlabeled": false } } } @@ -162,23 +163,27 @@ GET /twitter/tweet/_rank_eval // CONSOLE // TEST[setup:twitter] -<1> For graded relevance ratings only ratings above this threshold are -considered as relevant results for the given query. By default this is set to 1. +The `precision` metric takes the following optional parameters -<2> All documents retrieved by the rated request that have no ratings -assigned are treated unrelevant by default. Set to true in order to drop them -from the precision computation entirely. +[cols="<,<",options="header",] +|======================================================================= +|Parameter |Description +|`relevant_rating_threshold` |Sets the rating threshold from which on documents are considered to be +"relevant". Defaults to `1`. +|`ignore_unlabeled` |controls how unlabeled documents in the search results are counted. +If set to 'true', unlabeled documents are ignored and neither count as relevant or irrelevant. Set to 'false' (the default), they are treated as irrelevant. +|======================================================================= +=== Mean reciprocal rank -=== Reciprocal rank - -For any given query this is the reciprocal of the rank of the -first relevant document retrieved. For example finding the first relevant result -in position 3 means Reciprocal Rank is going to be 1/3. +For every query in the test suite, this metric calculates the reciprocal of the rank of the +first relevant document. For example finding the first relevant result +in position 3 means the reciprocal rank is 1/3. The reciprocal rank for each query +is averaged across all queries in the test suite to give the https://en.wikipedia.org/wiki/Mean_reciprocal_rank[mean reciprocal rank]. [source,js] -------------------------------- -GET /twitter/tweet/_rank_eval +GET /twitter/_rank_eval { "requests": [ { @@ -187,25 +192,31 @@ GET /twitter/tweet/_rank_eval "ratings": [] }], "metric": { - "reciprocal_rank": {} + "mean_reciprocal_rank": {} } } -------------------------------- // CONSOLE // TEST[setup:twitter] -=== Normalized discounted cumulative gain +The `mean_reciprocal_rank` metric takes the following optional parameters -In contrast to the two metrics above this takes both, the grade of the result -found as well as the position of the document returned into account. +[cols="<,<",options="header",] +|======================================================================= +|Parameter |Description +|`relevant_rating_threshold` |Sets the rating threshold from which on documents are considered to be +"relevant". Defaults to `1`. +|======================================================================= -For more details also check the explanation on -https://en.wikipedia.org/wiki/Discounted_cumulative_gain[Wikipedia]. +=== Discounted cumulative gain (DCG) +In contrast to the two metrics above, https://en.wikipedia.org/wiki/Discounted_cumulative_gain[discounted cumulative gain] takes both, the rank and the rating of the search results, into account. + +The assumption is that highly relevant documents are more useful for the user when appearing at the top of the result list. Therefore, the DCG formula reduces the contribution that high ratings for documents on lower search ranks have on the overall DCG metric. [source,js] -------------------------------- -GET /twitter/tweet/_rank_eval +GET /twitter/_rank_eval { "requests": [ { @@ -215,7 +226,7 @@ GET /twitter/tweet/_rank_eval }], "metric": { "dcg": { - "normalize": false <1> + "normalize": false } } } @@ -223,9 +234,15 @@ GET /twitter/tweet/_rank_eval // CONSOLE // TEST[setup:twitter] -<1> Set to true to compute nDCG instead of DCG, default is false. +The `dcg` metric takes the following optional parameters: + +[cols="<,<",options="header",] +|======================================================================= +|Parameter |Description +|`normalize` | If set to `true`, this metric will calculate the https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG[Normalized DCG]. +|======================================================================= + +== Other parameters + +== Response format -Setting normalize to true makes DCG values better comparable across different -result set sizes. See also -https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG[Wikipedia -nDCG] for more details. diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java similarity index 88% rename from modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java rename to modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java index 17699de1d40..c8a84a03b39 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java @@ -37,13 +37,13 @@ import javax.naming.directory.SearchResult; import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; /** - * Evaluate reciprocal rank. By default documents with a rating equal or bigger + * Evaluate mean reciprocal rank. By default documents with a rating equal or bigger * than 1 are considered to be "relevant" for the reciprocal rank calculation. * This value can be changes using the "relevant_rating_threshold" parameter. */ -public class ReciprocalRank implements RankedListQualityMetric { +public class MeanReciprocalRank implements RankedListQualityMetric { - public static final String NAME = "reciprocal_rank"; + public static final String NAME = "mean_reciprocal_rank"; /** ratings equal or above this value will be considered relevant. */ private int relevantRatingThreshhold = 1; @@ -51,11 +51,11 @@ public class ReciprocalRank implements RankedListQualityMetric { /** * Initializes maxAcceptableRank with 10 */ - public ReciprocalRank() { + public MeanReciprocalRank() { // use defaults } - public ReciprocalRank(StreamInput in) throws IOException { + public MeanReciprocalRank(StreamInput in) throws IOException { this.relevantRatingThreshhold = in.readVInt(); } @@ -121,14 +121,14 @@ public class ReciprocalRank implements RankedListQualityMetric { private static final ParseField RELEVANT_RATING_FIELD = new ParseField( "relevant_rating_threshold"); - private static final ObjectParser PARSER = new ObjectParser<>( - "reciprocal_rank", () -> new ReciprocalRank()); + private static final ObjectParser PARSER = new ObjectParser<>( + "reciprocal_rank", () -> new MeanReciprocalRank()); static { - PARSER.declareInt(ReciprocalRank::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); + PARSER.declareInt(MeanReciprocalRank::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); } - public static ReciprocalRank fromXContent(XContentParser parser) { + public static MeanReciprocalRank fromXContent(XContentParser parser) { return PARSER.apply(parser, null); } @@ -150,7 +150,7 @@ public class ReciprocalRank implements RankedListQualityMetric { if (obj == null || getClass() != obj.getClass()) { return false; } - ReciprocalRank other = (ReciprocalRank) obj; + MeanReciprocalRank other = (MeanReciprocalRank) obj; return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold); } @@ -200,7 +200,7 @@ public class ReciprocalRank implements RankedListQualityMetric { if (obj == null || getClass() != obj.getClass()) { return false; } - ReciprocalRank.Breakdown other = (ReciprocalRank.Breakdown) obj; + MeanReciprocalRank.Breakdown other = (MeanReciprocalRank.Breakdown) obj; return Objects.equals(firstRelevantRank, other.firstRelevantRank); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java similarity index 91% rename from modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java rename to modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java index 72055b01137..ea31573f8b3 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Precision.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java @@ -42,15 +42,15 @@ import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsW * considered to be "relevant" for the precision calculation. This value can be * changes using the "relevant_rating_threshold" parameter. */ -public class Precision implements RankedListQualityMetric { +public class PrecisionAtK implements RankedListQualityMetric { public static final String NAME = "precision"; private static final ParseField RELEVANT_RATING_FIELD = new ParseField( "relevant_rating_threshold"); private static final ParseField IGNORE_UNLABELED_FIELD = new ParseField("ignore_unlabeled"); - private static final ObjectParser PARSER = new ObjectParser<>(NAME, - Precision::new); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, + PrecisionAtK::new); /** * This setting controls how unlabeled documents in the search hits are @@ -63,16 +63,16 @@ public class Precision implements RankedListQualityMetric { /** ratings equal or above this value will be considered relevant. */ private int relevantRatingThreshhold = 1; - public Precision() { + public PrecisionAtK() { // needed for supplier in parser } static { - PARSER.declareInt(Precision::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); - PARSER.declareBoolean(Precision::setIgnoreUnlabeled, IGNORE_UNLABELED_FIELD); + PARSER.declareInt(PrecisionAtK::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); + PARSER.declareBoolean(PrecisionAtK::setIgnoreUnlabeled, IGNORE_UNLABELED_FIELD); } - public Precision(StreamInput in) throws IOException { + public PrecisionAtK(StreamInput in) throws IOException { relevantRatingThreshhold = in.readOptionalVInt(); ignoreUnlabeled = in.readOptionalBoolean(); } @@ -122,7 +122,7 @@ public class Precision implements RankedListQualityMetric { return ignoreUnlabeled; } - public static Precision fromXContent(XContentParser parser) { + public static PrecisionAtK fromXContent(XContentParser parser) { return PARSER.apply(parser, null); } @@ -155,7 +155,7 @@ public class Precision implements RankedListQualityMetric { } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision); evalQueryQuality.addMetricDetails( - new Precision.Breakdown(truePositives, truePositives + falsePositives)); + new PrecisionAtK.Breakdown(truePositives, truePositives + falsePositives)); evalQueryQuality.addHitsAndRatings(ratedSearchHits); return evalQueryQuality; } @@ -179,7 +179,7 @@ public class Precision implements RankedListQualityMetric { if (obj == null || getClass() != obj.getClass()) { return false; } - Precision other = (Precision) obj; + PrecisionAtK other = (PrecisionAtK) obj; return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold) && Objects.equals(ignoreUnlabeled, other.ignoreUnlabeled); } @@ -241,7 +241,7 @@ public class Precision implements RankedListQualityMetric { if (obj == null || getClass() != obj.getClass()) { return false; } - Precision.Breakdown other = (Precision.Breakdown) obj; + PrecisionAtK.Breakdown other = (PrecisionAtK.Breakdown) obj; return Objects.equals(relevantRetrieved, other.relevantRetrieved) && Objects.equals(retrieved, other.retrieved); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index 7bc291e35da..69825405363 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -65,15 +65,15 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin { public List getNamedWriteables() { List namedWriteables = new ArrayList<>(); namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, - Precision.NAME, Precision::new)); + PrecisionAtK.NAME, PrecisionAtK::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, - ReciprocalRank.NAME, ReciprocalRank::new)); + MeanReciprocalRank.NAME, MeanReciprocalRank::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, Precision.NAME, - Precision.Breakdown::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, PrecisionAtK.NAME, + PrecisionAtK.Breakdown::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, - ReciprocalRank.NAME, ReciprocalRank.Breakdown::new)); + MeanReciprocalRank.NAME, MeanReciprocalRank.Breakdown::new)); return namedWriteables; } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index cf1da4388fe..35fa49fa7fa 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -65,11 +65,11 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { // TODO maybe switch to using a plugable registry later? switch (metricName) { - case Precision.NAME: - rc = Precision.fromXContent(parser); + case PrecisionAtK.NAME: + rc = PrecisionAtK.fromXContent(parser); break; - case ReciprocalRank.NAME: - rc = ReciprocalRank.fromXContent(parser); + case MeanReciprocalRank.NAME: + rc = MeanReciprocalRank.fromXContent(parser); break; case DiscountedCumulativeGain.NAME: rc = DiscountedCumulativeGain.fromXContent(parser); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 5edd1061936..c652f7d5e3a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -36,119 +36,57 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; /** - * Accepted input format: - * - * General Format: - * - * - { - "spec_id": "human_readable_id", - "requests": [{ - "id": "human_readable_id", - "request": { ... request to check ... }, - "ratings": { ... mapping from doc id to rating value ... } - }], - "metric": { - "... metric_name... ": { - "... metric_parameter_key ...": ...metric_parameter_value... - } - } - } - * - * Example: - * - * - {"spec_id": "huge_weight_on_location", - "requests": [{ - "id": "amsterdam_query", - "request": { - "query": { - "bool": { - "must": [ - {"match": {"beverage": "coffee"}}, - {"term": {"browser": {"value": "safari"}}}, - {"term": {"time_of_day": {"value": "morning","boost": 2}}}, - {"term": {"ip_location": {"value": "ams","boost": 10}}}]} - }, - "size": 10 - }, - "ratings": [ - {\"index\": \"test\", \"type\": \"my_type\", \"doc_id\": \"1\", \"rating\" : 1 }, - {\"index\": \"test\", \"type\": \"my_type\", \"doc_id\": \"2\", \"rating\" : 0 }, - {\"index\": \"test\", \"type\": \"my_type\", \"doc_id\": \"3\", \"rating\" : 1 } - ] - }, { - "id": "berlin_query", - "request": { - "query": { - "bool": { - "must": [ - {"match": {"beverage": "club mate"}}, - {"term": {"browser": {"value": "chromium"}}}, - {"term": {"time_of_day": {"value": "evening","boost": 2}}}, - {"term": {"ip_location": {"value": "ber","boost": 10}}}]} - }, - "size": 10 - }, - "ratings": [ ... ] - }], - "metric": { - "precisionAtN": { - "size": 10 - } - } - } - - * - * Output format: - * - * General format: - * - * - { - "took": 59, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "failed": 0 - }, - "quality_level": ... quality level ..., - "unknown_docs": {"user_request_id": [... list of unknown docs ...]} -} - - * - * Example: - * - * - * - { - "took": 59, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "failed": 0 - }, - "rank_eval": [{ - "spec_id": "huge_weight_on_location", - "quality_level": 0.4, - "unknown_docs": { - "amsterdam_query": [ - { "index" : "test", "doc_id" : "21"}, - { "index" : "test", "doc_id" : "5"}, - { "index" : "test", "doc_id" : "9"} - ] - }, { - "berlin_query": [ - { "index" : "test", "doc_id" : "42"} - ] - } - }] - } - - - * */ + * { + * "requests": [{ + * "id": "amsterdam_query", + * "request": { + * "query": { + * "match": { + * "text": "amsterdam" + * } + * } + * }, + * "ratings": [{ + * "_index": "foo", + * "_id": "doc1", + * "rating": 0 + * }, + * { + * "_index": "foo", + * "_id": "doc2", + * "rating": 1 + * }, + * { + * "_index": "foo", + * "_id": "doc3", + * "rating": 1 + * } + * ] + * }, + * { + * "id": "berlin_query", + * "request": { + * "query": { + * "match": { + * "text": "berlin" + * } + * }, + * "size": 10 + * }, + * "ratings": [{ + * "_index": "foo", + * "_id": "doc1", + * "rating": 1 + * }] + * } + * ], + * "metric": { + * "precision": { + * "ignore_unlabeled": true + * } + * } + * } + */ public class RestRankEvalAction extends BaseRestHandler { public RestRankEvalAction(Settings settings, RestController controller) { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java index b938dd91d12..97e40f12f30 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -46,7 +46,7 @@ public class EvalQueryQualityTests extends ESTestCase { randomDoubleBetween(0.0, 1.0, true)); if (randomBoolean()) { // TODO randomize this - evalQueryQuality.addMetricDetails(new Precision.Breakdown(1, 5)); + evalQueryQuality.addMetricDetails(new PrecisionAtK.Breakdown(1, 5)); } evalQueryQuality.addHitsAndRatings(ratedHits); return evalQueryQuality; @@ -81,7 +81,7 @@ public class EvalQueryQualityTests extends ESTestCase { break; case 2: if (metricDetails == null) { - metricDetails = new Precision.Breakdown(1, 5); + metricDetails = new PrecisionAtK.Breakdown(1, 5); } else { metricDetails = null; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java index a520de378b9..d520174c28e 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java @@ -43,10 +43,10 @@ public class PrecisionTests extends ESTestCase { public void testPrecisionAtFiveCalculation() { List rated = new ArrayList<>(); rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal())); - EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated, "test"), rated); + EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated, "test"), rated); assertEquals(1, evaluated.getQualityLevel(), 0.00001); - assertEquals(1, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(1, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(1, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(1, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveIgnoreOneResult() { @@ -56,10 +56,10 @@ public class PrecisionTests extends ESTestCase { rated.add(new RatedDocument("test", "2", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "3", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "4", Rating.IRRELEVANT.ordinal())); - EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated, "test"), rated); + EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated, "test"), rated); assertEquals((double) 4 / 5, evaluated.getQualityLevel(), 0.00001); - assertEquals(4, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(4, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(5, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } /** @@ -74,12 +74,12 @@ public class PrecisionTests extends ESTestCase { rated.add(new RatedDocument("test", "2", 2)); rated.add(new RatedDocument("test", "3", 3)); rated.add(new RatedDocument("test", "4", 4)); - Precision precisionAtN = new Precision(); + PrecisionAtK precisionAtN = new PrecisionAtK(); precisionAtN.setRelevantRatingThreshhold(2); EvalQueryQuality evaluated = precisionAtN.evaluate("id", toSearchHits(rated, "test"), rated); assertEquals((double) 3 / 5, evaluated.getQualityLevel(), 0.00001); - assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(3, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(5, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveCorrectIndex() { @@ -90,10 +90,10 @@ public class PrecisionTests extends ESTestCase { rated.add(new RatedDocument("test", "1", Rating.RELEVANT.ordinal())); rated.add(new RatedDocument("test", "2", Rating.IRRELEVANT.ordinal())); // the following search hits contain only the last three documents - EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated.subList(2, 5), "test"), rated); + EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated.subList(2, 5), "test"), rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(2, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(3, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testIgnoreUnlabeled() { @@ -105,18 +105,18 @@ public class PrecisionTests extends ESTestCase { searchHits[2] = new SearchHit(2, "2", new Text("testtype"), Collections.emptyMap()); searchHits[2].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); - EvalQueryQuality evaluated = (new Precision()).evaluate("id", searchHits, rated); + EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", searchHits, rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(2, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(3, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); // also try with setting `ignore_unlabeled` - Precision prec = new Precision(); + PrecisionAtK prec = new PrecisionAtK(); prec.setIgnoreUnlabeled(true); evaluated = prec.evaluate("id", searchHits, rated); assertEquals((double) 2 / 2, evaluated.getQualityLevel(), 0.00001); - assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(2, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(2, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testNoRatedDocs() throws Exception { @@ -125,30 +125,30 @@ public class PrecisionTests extends ESTestCase { hits[i] = new SearchHit(i, i + "", new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); } - EvalQueryQuality evaluated = (new Precision()).evaluate("id", hits, Collections.emptyList()); + EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", hits, Collections.emptyList()); assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); - assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(5, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); // also try with setting `ignore_unlabeled` - Precision prec = new Precision(); + PrecisionAtK prec = new PrecisionAtK(); prec.setIgnoreUnlabeled(true); evaluated = prec.evaluate("id", hits, Collections.emptyList()); assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); - assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); - assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testParseFromXContent() throws IOException { String xContent = " {\n" + " \"relevant_rating_threshold\" : 2" + "}"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) { - Precision precicionAt = Precision.fromXContent(parser); + PrecisionAtK precicionAt = PrecisionAtK.fromXContent(parser); assertEquals(2, precicionAt.getRelevantRatingThreshold()); } } public void testCombine() { - Precision metric = new Precision(); + PrecisionAtK metric = new PrecisionAtK(); Vector partialResults = new Vector<>(3); partialResults.add(new EvalQueryQuality("a", 0.1)); partialResults.add(new EvalQueryQuality("b", 0.2)); @@ -157,12 +157,12 @@ public class PrecisionTests extends ESTestCase { } public void testInvalidRelevantThreshold() { - Precision prez = new Precision(); + PrecisionAtK prez = new PrecisionAtK(); expectThrows(IllegalArgumentException.class, () -> prez.setRelevantRatingThreshhold(-1)); } - public static Precision createTestItem() { - Precision precision = new Precision(); + public static PrecisionAtK createTestItem() { + PrecisionAtK precision = new PrecisionAtK(); if (randomBoolean()) { precision.setRelevantRatingThreshhold(randomIntBetween(0, 10)); } @@ -171,13 +171,13 @@ public class PrecisionTests extends ESTestCase { } public void testXContentRoundtrip() throws IOException { - Precision testItem = createTestItem(); + PrecisionAtK testItem = createTestItem(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); itemParser.nextToken(); - Precision parsedItem = Precision.fromXContent(itemParser); + PrecisionAtK parsedItem = PrecisionAtK.fromXContent(itemParser); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); @@ -185,22 +185,22 @@ public class PrecisionTests extends ESTestCase { } public void testSerialization() throws IOException { - Precision original = createTestItem(); - Precision deserialized = RankEvalTestHelper.copy(original, Precision::new); + PrecisionAtK original = createTestItem(); + PrecisionAtK deserialized = RankEvalTestHelper.copy(original, PrecisionAtK::new); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); } public void testEqualsAndHash() throws IOException { - Precision testItem = createTestItem(); - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), RankEvalTestHelper.copy(testItem, Precision::new)); + PrecisionAtK testItem = createTestItem(); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), RankEvalTestHelper.copy(testItem, PrecisionAtK::new)); } - private static Precision mutateTestItem(Precision original) { + private static PrecisionAtK mutateTestItem(PrecisionAtK original) { boolean ignoreUnlabeled = original.getIgnoreUnlabeled(); int relevantThreshold = original.getRelevantRatingThreshold(); - Precision precision = new Precision(); + PrecisionAtK precision = new PrecisionAtK(); precision.setIgnoreUnlabeled(ignoreUnlabeled); precision.setRelevantRatingThreshhold(relevantThreshold); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 22389ee7eec..2e3f86543c6 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -82,7 +82,7 @@ public class RankEvalRequestIT extends ESIntegTestCase { specifications.add(berlinRequest); - Precision metric = new Precision(); + PrecisionAtK metric = new PrecisionAtK(); metric.setIgnoreUnlabeled(true); RankEvalSpec task = new RankEvalSpec(specifications, metric); @@ -148,7 +148,7 @@ public class RankEvalRequestIT extends ESIntegTestCase { brokenRequest.setIndices(indices); specifications.add(brokenRequest); - RankEvalSpec task = new RankEvalSpec(specifications, new Precision()); + RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK()); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 8a365672a24..40624f12dab 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -114,10 +114,11 @@ public class RankEvalSpecTests extends ESTestCase { List namedWriteables = new ArrayList<>(); namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, PrecisionAtK.NAME, PrecisionAtK::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); + namedWriteables + .add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, MeanReciprocalRank.NAME, MeanReciprocalRank::new)); RankEvalSpec deserialized = RankEvalTestHelper.copy(original, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables)); assertEquals(deserialized, original); @@ -130,10 +131,11 @@ public class RankEvalSpecTests extends ESTestCase { List namedWriteables = new ArrayList<>(); namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, PrecisionAtK.NAME, PrecisionAtK::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); + namedWriteables + .add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, MeanReciprocalRank.NAME, MeanReciprocalRank::new)); RankEvalSpec mutant = RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables)); RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(mutant), @@ -152,10 +154,10 @@ public class RankEvalSpecTests extends ESTestCase { ratedRequests.add(request); break; case 1: - if (metric instanceof Precision) { + if (metric instanceof PrecisionAtK) { metric = new DiscountedCumulativeGain(); } else { - metric = new Precision(); + metric = new PrecisionAtK(); } break; case 2: @@ -175,7 +177,7 @@ public class RankEvalSpecTests extends ESTestCase { } public void testMissingRatedRequestsFailsParsing() { - RankedListQualityMetric metric = new Precision(); + RankedListQualityMetric metric = new PrecisionAtK(); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(new ArrayList<>(), metric)); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(null, metric)); } @@ -194,6 +196,6 @@ public class RankEvalSpecTests extends ESTestCase { RatedRequest request = new RatedRequest("id", ratedDocs, params, "templateId"); List ratedRequests = Arrays.asList(request); - expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, new Precision())); + expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, new PrecisionAtK())); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 273a2ff7a5c..bed24a18505 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -41,7 +41,7 @@ import java.util.Vector; public class ReciprocalRankTests extends ESTestCase { public void testMaxAcceptableRank() { - ReciprocalRank reciprocalRank = new ReciprocalRank(); + MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(); int searchHits = randomIntBetween(1, 50); @@ -59,17 +59,17 @@ public class ReciprocalRankTests extends ESTestCase { int rankAtFirstRelevant = relevantAt + 1; EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE); - assertEquals(rankAtFirstRelevant, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); + assertEquals(rankAtFirstRelevant, ((MeanReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); // check that if we have fewer search hits than relevant doc position, // we don't find any result and get 0.0 quality level - reciprocalRank = new ReciprocalRank(); + reciprocalRank = new MeanReciprocalRank(); evaluation = reciprocalRank.evaluate("id", Arrays.copyOfRange(hits, 0, relevantAt), ratedDocs); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); } public void testEvaluationOneRelevantInResults() { - ReciprocalRank reciprocalRank = new ReciprocalRank(); + MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(); SearchHit[] hits = createSearchHits(0, 9, "test"); List ratedDocs = new ArrayList<>(); // mark one of the ten docs relevant @@ -84,7 +84,7 @@ public class ReciprocalRankTests extends ESTestCase { EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE); - assertEquals(relevantAt + 1, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); + assertEquals(relevantAt + 1, ((MeanReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } /** @@ -101,15 +101,15 @@ public class ReciprocalRankTests extends ESTestCase { rated.add(new RatedDocument("test", "4", 4)); SearchHit[] hits = createSearchHits(0, 5, "test"); - ReciprocalRank reciprocalRank = new ReciprocalRank(); + MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(); reciprocalRank.setRelevantRatingThreshhold(2); EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, rated); assertEquals((double) 1 / 3, evaluation.getQualityLevel(), 0.00001); - assertEquals(3, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); + assertEquals(3, ((MeanReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } public void testCombine() { - ReciprocalRank reciprocalRank = new ReciprocalRank(); + MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(); Vector partialResults = new Vector<>(3); partialResults.add(new EvalQueryQuality("id1", 0.5)); partialResults.add(new EvalQueryQuality("id2", 1.0)); @@ -118,7 +118,7 @@ public class ReciprocalRankTests extends ESTestCase { } public void testEvaluationNoRelevantInResults() { - ReciprocalRank reciprocalRank = new ReciprocalRank(); + MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(); SearchHit[] hits = createSearchHits(0, 9, "test"); List ratedDocs = new ArrayList<>(); EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); @@ -126,13 +126,13 @@ public class ReciprocalRankTests extends ESTestCase { } public void testXContentRoundtrip() throws IOException { - ReciprocalRank testItem = createTestItem(); + MeanReciprocalRank testItem = createTestItem(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); itemParser.nextToken(); - ReciprocalRank parsedItem = ReciprocalRank.fromXContent(itemParser); + MeanReciprocalRank parsedItem = MeanReciprocalRank.fromXContent(itemParser); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); @@ -152,36 +152,36 @@ public class ReciprocalRankTests extends ESTestCase { return hits; } - private static ReciprocalRank createTestItem() { - ReciprocalRank testItem = new ReciprocalRank(); + private static MeanReciprocalRank createTestItem() { + MeanReciprocalRank testItem = new MeanReciprocalRank(); testItem.setRelevantRatingThreshhold(randomIntBetween(0, 20)); return testItem; } public void testSerialization() throws IOException { - ReciprocalRank original = createTestItem(); + MeanReciprocalRank original = createTestItem(); - ReciprocalRank deserialized = RankEvalTestHelper.copy(original, ReciprocalRank::new); + MeanReciprocalRank deserialized = RankEvalTestHelper.copy(original, MeanReciprocalRank::new); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); } public void testEqualsAndHash() throws IOException { - ReciprocalRank testItem = createTestItem(); + MeanReciprocalRank testItem = createTestItem(); RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - RankEvalTestHelper.copy(testItem, ReciprocalRank::new)); + RankEvalTestHelper.copy(testItem, MeanReciprocalRank::new)); } - private static ReciprocalRank mutateTestItem(ReciprocalRank testItem) { + private static MeanReciprocalRank mutateTestItem(MeanReciprocalRank testItem) { int relevantThreshold = testItem.getRelevantRatingThreshold(); - ReciprocalRank rank = new ReciprocalRank(); + MeanReciprocalRank rank = new MeanReciprocalRank(); rank.setRelevantRatingThreshhold(randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10))); return rank; } public void testInvalidRelevantThreshold() { - ReciprocalRank prez = new ReciprocalRank(); + MeanReciprocalRank prez = new MeanReciprocalRank(); expectThrows(IllegalArgumentException.class, () -> prez.setRelevantRatingThreshhold(-1)); } } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml index 991b4c84dc0..1c5b82468cc 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml @@ -82,7 +82,7 @@ - is_false: rank_eval.details.berlin_query.hits.1.rating --- -"Reciprocal Rank": +"Mean Reciprocal Rank": - do: indices.create: @@ -139,7 +139,7 @@ "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}] } ], - "metric" : { "reciprocal_rank": {} } + "metric" : { "mean_reciprocal_rank": {} } } # average is (1/3 + 1/2)/2 = 5/12 ~ 0.41666666666666663 From 5c65a59369950b5ddfbb24ff4fd79a0234352736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 13 Nov 2017 21:37:29 +0100 Subject: [PATCH 108/297] Extending rank_eval asciidocs --- docs/reference/search/rank-eval.asciidoc | 64 ++++++++++++++++--- .../test/rank_eval/30_failures.yml | 2 +- 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index 321791e6b03..886d262cc72 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -29,7 +29,7 @@ In order to get started with search quality evaluation, three basic things a nee The ranking evaluation API provides a convenient way to use this information in a ranking evaluation request to calculate different search evaluation metrics. This gives a first estimation of your overall search quality and give you a measurement to optimize against when fine-tuning various aspect of the query generation in your application. -== Ranking evaluation request structure +=== Ranking evaluation request structure In its most basic form, a request to the `_rank_eval` endpoint has two sections: @@ -85,7 +85,7 @@ the rating of the documents relevance with regards to this search request A document `rating` can be any integer value that expresses the relevance of the document on a user defined scale. For some of the metrics, just giving a binary rating (e.g. `0` for irrelevant and `1` for relevant) will be sufficient, other metrics can use a more fine grained scale. -== Template based ranking evaluation +=== Template based ranking evaluation As an alternative to having to provide a single query per test request, it is possible to specify query templates in the evaluation request and later refer to them. Queries with similar structure that only differ in their parameters don't have to be repeated all the time in the `requests` section this way. In typical search systems where user inputs usually get filled into a small set of query templates, this helps making the evaluation request more succinct. @@ -126,12 +126,12 @@ GET /my_index/_rank_eval <3> a reference to a previously defined temlate <4> the parameters to use to fill the template -== Available evaluation metrics +=== Available evaluation metrics The `metric` section determines which of the available evaluation metrics is going to be used. Currently, the following metrics are supported: -=== Precision at k (Prec@k) +==== Precision at k (Prec@k) This metric measures the number of relevant results in the top k search results. Its a form of the well known https://en.wikipedia.org/wiki/Information_retrieval#Precision[Precision] metric that only looks at the top k documents. It is the fraction of relevant documents in those first k search. A precision at 10 (prec@10) value of 0.6 then means six out of the 10 top hits where @@ -174,7 +174,7 @@ The `precision` metric takes the following optional parameters If set to 'true', unlabeled documents are ignored and neither count as relevant or irrelevant. Set to 'false' (the default), they are treated as irrelevant. |======================================================================= -=== Mean reciprocal rank +==== Mean reciprocal rank For every query in the test suite, this metric calculates the reciprocal of the rank of the first relevant document. For example finding the first relevant result @@ -208,7 +208,7 @@ The `mean_reciprocal_rank` metric takes the following optional parameters "relevant". Defaults to `1`. |======================================================================= -=== Discounted cumulative gain (DCG) +==== Discounted cumulative gain (DCG) In contrast to the two metrics above, https://en.wikipedia.org/wiki/Discounted_cumulative_gain[discounted cumulative gain] takes both, the rank and the rating of the search results, into account. @@ -242,7 +242,55 @@ The `dcg` metric takes the following optional parameters: |`normalize` | If set to `true`, this metric will calculate the https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG[Normalized DCG]. |======================================================================= -== Other parameters +=== Response format -== Response format +The response of the `_rank_eval` endpoint contains the overall calculated result for the defined quality metric, +a `details` section with a breakdown of results for each query in the test suite and an optional `failures` section +that shows potential errors of individual queries. The response has the following format: +[source,js] +-------------------------------- +{ + "rank_eval": { + "quality_level": 0.4, <1> + "details": { + "my_query_id1": { <2> + "quality_level": 0.6, <3> + "unknown_docs": [ <4> + { + "_index": "my_index", + "_id": "1960795" + }, [...] + ], + "hits": [ + { + "hit": { <5> + "_index": "my_index", + "_type": "page", + "_id": "1528558", + "_score": 7.0556192 + }, + "rating": 1 + }, [...] + ], + "metric_details": { <6> + "relevant_docs_retrieved": 6, + "docs_retrieved": 10 + } + }, + "my_query_id2 : { [...]} + }, + "failures": { [...] } + } +} +-------------------------------- +// NOTCONSOLE + +<1> the overall evaluation quality calculated by the defined metric +<2> the `details` section contains one entry for every query in the original `requests` section, keyed by the search request id +<3> the `quality_level` in the `details` section shows the contribution of this query to the global quality score +<4> the `unknown_docs` section contains an `_index` and `_id` entry for each document in the search result for this +query that didn't have a ratings value. This can be used to ask the user to supply ratings for these documents +<5> the `hits` section shows a grouping of the search results with their supplied rating +<6> the `metric_details` give additional information about the calculated quality metric (e.g. how many of the retrieved +documents where relevant). The content varies for each metric but allows for better interpretation of the results diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml index 130da28f3b1..0df0110993a 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml @@ -35,4 +35,4 @@ - match: { rank_eval.details.amsterdam_query.unknown_docs: [ ]} - match: { rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}} - - is_true: rank_eval.failures.invalid_query + - is_true: rank_eval.failures.invalid_queryy From e278c1d17dd6276fe2fc65495691077f6826ab4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 14 Nov 2017 19:26:32 +0100 Subject: [PATCH 109/297] Improving and cleaning up tests Removing the unnecessary RankEvalTestHelper, making use of the common test infra in ESTestCase, also hardening a few of the classes by making more fields final. --- .../rankeval/DiscountedCumulativeGain.java | 48 +++----- .../index/rankeval/DocumentKey.java | 106 ---------------- .../index/rankeval/EvalQueryQuality.java | 8 +- ...alityMetric.java => EvaluationMetric.java} | 23 ++-- .../index/rankeval/MeanReciprocalRank.java | 56 +++++---- .../index/rankeval/PrecisionAtK.java | 69 +++++------ .../index/rankeval/RankEvalPlugin.java | 21 ++-- .../index/rankeval/RankEvalSpec.java | 24 ++-- .../index/rankeval/RatedDocument.java | 101 +++++++++++---- .../index/rankeval/RatedRequest.java | 1 + .../rankeval/TransportRankEvalAction.java | 4 +- .../DiscountedCumulativeGainTests.java | 115 ++++++++++-------- .../index/rankeval/DocumentKeyTests.java | 67 ---------- .../index/rankeval/EvalQueryQualityTests.java | 19 +-- ...ests.java => MeanReciprocalRankTests.java} | 60 +++++---- ...isionTests.java => PrecisionAtKTests.java} | 90 +++++++------- .../index/rankeval/RankEvalRequestIT.java | 12 +- .../index/rankeval/RankEvalResponseTests.java | 12 +- .../index/rankeval/RankEvalSpecTests.java | 51 ++++---- .../index/rankeval/RankEvalTestHelper.java | 94 -------------- .../index/rankeval/RatedDocumentTests.java | 25 ++-- .../index/rankeval/RatedRequestsTests.java | 40 +++--- .../index/rankeval/RatedSearchHitTests.java | 14 ++- .../index/rankeval/TestRatingEnum.java | 24 ++++ .../test/rank_eval/30_failures.yml | 2 +- 25 files changed, 450 insertions(+), 636 deletions(-) delete mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java rename modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/{RankedListQualityMetric.java => EvaluationMetric.java} (83%) delete mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{ReciprocalRankTests.java => MeanReciprocalRankTests.java} (79%) rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{PrecisionTests.java => PrecisionAtKTests.java} (74%) delete mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TestRatingEnum.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java index 6f843f92461..a544ffcb4ea 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java @@ -22,7 +22,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; @@ -35,22 +35,25 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; +import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings; -public class DiscountedCumulativeGain implements RankedListQualityMetric { +public class DiscountedCumulativeGain implements EvaluationMetric { /** If set to true, the dcg will be normalized (ndcg) */ - private boolean normalize; + private final boolean normalize; + /** * If set to, this will be the rating for docs the user hasn't supplied an * explicit rating for */ - private Integer unknownDocRating; + private final Integer unknownDocRating; public static final String NAME = "dcg"; private static final double LOG2 = Math.log(2.0); public DiscountedCumulativeGain() { + this(false, null); } /** @@ -82,13 +85,6 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { return NAME; } - /** - * If set to true, the dcg will be normalized (ndcg) - */ - public void setNormalize(boolean normalize) { - this.normalize = normalize; - } - /** * check whether this metric computes only dcg or "normalized" ndcg */ @@ -96,13 +92,6 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { return this.normalize; } - /** - * the rating for docs the user hasn't supplied an explicit rating for - */ - public void setUnknownDocRating(int unknownDocRating) { - this.unknownDocRating = unknownDocRating; - } - /** * get the rating used for unrated documents */ @@ -118,10 +107,10 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { List ratedHits = joinHitsWithRatings(hits, ratedDocs); List ratingsInSearchHits = new ArrayList<>(ratedHits.size()); for (RatedSearchHit hit : ratedHits) { - // unknownDocRating might be null, which means it will be unrated - // docs are ignored in the dcg calculation - // we still need to add them as a placeholder so the rank of the - // subsequent ratings is correct + // unknownDocRating might be null, which means it will be unrated docs are + // ignored in the dcg calculation + // we still need to add them as a placeholder so the rank of the subsequent + // ratings is correct ratingsInSearchHits.add(hit.getRating().orElse(unknownDocRating)); } double dcg = computeDCG(ratingsInSearchHits); @@ -151,12 +140,15 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { private static final ParseField NORMALIZE_FIELD = new ParseField("normalize"); private static final ParseField UNKNOWN_DOC_RATING_FIELD = new ParseField("unknown_doc_rating"); - private static final ObjectParser PARSER = new ObjectParser<>( - "dcg_at", () -> new DiscountedCumulativeGain()); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("dcg_at", + args -> { + Boolean normalized = (Boolean) args[0]; + return new DiscountedCumulativeGain(normalized == null ? false : normalized, (Integer) args[1]); + }); static { - PARSER.declareBoolean(DiscountedCumulativeGain::setNormalize, NORMALIZE_FIELD); - PARSER.declareInt(DiscountedCumulativeGain::setUnknownDocRating, UNKNOWN_DOC_RATING_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), NORMALIZE_FIELD); + PARSER.declareInt(optionalConstructorArg(), UNKNOWN_DOC_RATING_FIELD); } public static DiscountedCumulativeGain fromXContent(XContentParser parser) { @@ -193,6 +185,4 @@ public class DiscountedCumulativeGain implements RankedListQualityMetric { public final int hashCode() { return Objects.hash(normalize, unknownDocRating); } - - // TODO maybe also add debugging breakdown here } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java deleted file mode 100644 index 7bbda60f18c..00000000000 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DocumentKey.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.rankeval; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Objects; - -public class DocumentKey implements Writeable, ToXContentObject { - - private String docId; - private String index; - - void setIndex(String index) { - this.index = index; - } - - void setDocId(String docId) { - this.docId = docId; - } - - public DocumentKey(String index, String docId) { - if (Strings.isNullOrEmpty(index)) { - throw new IllegalArgumentException("Index must be set for each rated document"); - } - if (Strings.isNullOrEmpty(docId)) { - throw new IllegalArgumentException("DocId must be set for each rated document"); - } - - this.index = index; - this.docId = docId; - } - - public DocumentKey(StreamInput in) throws IOException { - this.index = in.readString(); - this.docId = in.readString(); - } - - public String getIndex() { - return index; - } - - public String getDocID() { - return docId; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(index); - out.writeString(docId); - } - - @Override - public final boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - DocumentKey other = (DocumentKey) obj; - return Objects.equals(index, other.index) && Objects.equals(docId, other.docId); - } - - @Override - public final int hashCode() { - return Objects.hash(index, docId); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), index); - builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), docId); - builder.endObject(); - return builder; - } - - @Override - public String toString() { - return Strings.toString(this); - } -} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index 6cd7cbac52a..e7e00c1699d 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey; import java.io.IOException; import java.util.ArrayList; @@ -91,8 +92,11 @@ public class EvalQueryQuality implements ToXContent, Writeable { builder.startObject(id); builder.field("quality_level", this.qualityLevel); builder.startArray("unknown_docs"); - for (DocumentKey key : RankedListQualityMetric.filterUnknownDocuments(hits)) { - key.toXContent(builder, params); + for (DocumentKey key : EvaluationMetric.filterUnknownDocuments(hits)) { + builder.startObject(); + builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), key.getIndex()); + builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), key.getDocId()); + builder.endObject(); } builder.endArray(); builder.startArray("hits"); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java similarity index 83% rename from modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java rename to modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java index 35fa49fa7fa..6754d039266 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java @@ -24,7 +24,9 @@ import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import java.io.IOException; import java.util.ArrayList; @@ -35,13 +37,14 @@ import java.util.Optional; import java.util.stream.Collectors; /** - * Classes implementing this interface provide a means to compute the quality of a result list returned by some search. + * Implementations of {@link EvaluationMetric} need to provide a way to compute the quality metric for + * a result list returned by some search (@link {@link SearchHits}) and a list of rated documents. */ -public interface RankedListQualityMetric extends ToXContent, NamedWriteable { +public interface EvaluationMetric extends ToXContent, NamedWriteable { /** * Returns a single metric representing the ranking quality of a set of returned - * documents wrt. to a set of document Ids labeled as relevant for this search. + * documents wrt. to a set of document ids labeled as relevant for this search. * * @param taskId * the id of the query for which the ranking is currently evaluated @@ -55,15 +58,15 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { */ EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs); - static RankedListQualityMetric fromXContent(XContentParser parser) throws IOException { - RankedListQualityMetric rc; + static EvaluationMetric fromXContent(XContentParser parser) throws IOException { + EvaluationMetric rc; Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name"); } String metricName = parser.currentName(); - // TODO maybe switch to using a plugable registry later? + // TODO switch to using a plugable registry switch (metricName) { case PrecisionAtK.NAME: rc = PrecisionAtK.fromXContent(parser); @@ -101,13 +104,19 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable { return ratedSearchHits; } + /** + * filter @link {@link RatedSearchHit} that don't have a rating + */ static List filterUnknownDocuments(List ratedHits) { - // join hits with rated documents List unknownDocs = ratedHits.stream().filter(hit -> hit.getRating().isPresent() == false) .map(hit -> new DocumentKey(hit.getSearchHit().getIndex(), hit.getSearchHit().getId())).collect(Collectors.toList()); return unknownDocs; } + /** + * how evaluation metrics for particular search queries get combined for the overall evaluation score. + * Defaults to averaging over the partial results. + */ default double combine(Collection partialResults) { return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size(); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java index c8a84a03b39..2b47f68587e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java @@ -22,7 +22,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; @@ -34,49 +34,45 @@ import java.util.Optional; import javax.naming.directory.SearchResult; -import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; +import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings; /** - * Evaluate mean reciprocal rank. By default documents with a rating equal or bigger - * than 1 are considered to be "relevant" for the reciprocal rank calculation. - * This value can be changes using the "relevant_rating_threshold" parameter. + * Evaluates using mean reciprocal rank. By default documents with a rating + * equal or bigger than 1 are considered to be "relevant" for the reciprocal + * rank calculation. This value can be changes using the + * "relevant_rating_threshold" parameter. */ -public class MeanReciprocalRank implements RankedListQualityMetric { +public class MeanReciprocalRank implements EvaluationMetric { + + private static final int DEFAULT_RATING_THRESHOLD = 1; public static final String NAME = "mean_reciprocal_rank"; /** ratings equal or above this value will be considered relevant. */ - private int relevantRatingThreshhold = 1; + private final int relevantRatingThreshhold; - /** - * Initializes maxAcceptableRank with 10 - */ public MeanReciprocalRank() { - // use defaults + this(DEFAULT_RATING_THRESHOLD); } public MeanReciprocalRank(StreamInput in) throws IOException { this.relevantRatingThreshhold = in.readVInt(); } - @Override - public String getWriteableName() { - return NAME; - } - - /** - * Sets the rating threshold above which ratings are considered to be - * "relevant" for this metric. - */ - public void setRelevantRatingThreshhold(int threshold) { + public MeanReciprocalRank(int threshold) { if (threshold < 0) { throw new IllegalArgumentException( "Relevant rating threshold for precision must be positive integer."); } - this.relevantRatingThreshhold = threshold; } + @Override + public String getWriteableName() { + return NAME; + } + /** * Return the rating threshold above which ratings are considered to be * "relevant" for this metric. Defaults to 1. @@ -119,13 +115,19 @@ public class MeanReciprocalRank implements RankedListQualityMetric { out.writeVInt(relevantRatingThreshhold); } - private static final ParseField RELEVANT_RATING_FIELD = new ParseField( - "relevant_rating_threshold"); - private static final ObjectParser PARSER = new ObjectParser<>( - "reciprocal_rank", () -> new MeanReciprocalRank()); + private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("reciprocal_rank", + args -> { + Integer optionalThreshold = (Integer) args[0]; + if (optionalThreshold == null) { + return new MeanReciprocalRank(); + } else { + return new MeanReciprocalRank(optionalThreshold); + } + }); static { - PARSER.declareInt(MeanReciprocalRank::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); + PARSER.declareInt(optionalConstructorArg(), RELEVANT_RATING_FIELD); } public static MeanReciprocalRank fromXContent(XContentParser parser) { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java index ea31573f8b3..76cc72bc82d 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java @@ -22,7 +22,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; @@ -34,7 +34,8 @@ import java.util.Optional; import javax.naming.directory.SearchResult; -import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; +import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings; /** * Evaluate Precision of the search results. Documents without a rating are @@ -42,15 +43,12 @@ import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsW * considered to be "relevant" for the precision calculation. This value can be * changes using the "relevant_rating_threshold" parameter. */ -public class PrecisionAtK implements RankedListQualityMetric { +public class PrecisionAtK implements EvaluationMetric { public static final String NAME = "precision"; - private static final ParseField RELEVANT_RATING_FIELD = new ParseField( - "relevant_rating_threshold"); + private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); private static final ParseField IGNORE_UNLABELED_FIELD = new ParseField("ignore_unlabeled"); - private static final ObjectParser PARSER = new ObjectParser<>(NAME, - PrecisionAtK::new); /** * This setting controls how unlabeled documents in the search hits are @@ -58,29 +56,47 @@ public class PrecisionAtK implements RankedListQualityMetric { * as true or false positives. Set to 'false', they are treated as false * positives. */ - private boolean ignoreUnlabeled = false; + private final boolean ignoreUnlabeled; /** ratings equal or above this value will be considered relevant. */ - private int relevantRatingThreshhold = 1; + private final int relevantRatingThreshhold; + + public PrecisionAtK(int threshold, boolean ignoreUnlabeled) { + if (threshold < 0) { + throw new IllegalArgumentException( + "Relevant rating threshold for precision must be positive integer."); + } + this.relevantRatingThreshhold = threshold; + this.ignoreUnlabeled = ignoreUnlabeled; + } public PrecisionAtK() { - // needed for supplier in parser + this(1, false); } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, + args -> { + Integer threshHold = (Integer) args[0]; + Boolean ignoreUnlabeled = (Boolean) args[1]; + return new PrecisionAtK(threshHold == null ? 1 : threshHold, + ignoreUnlabeled == null ? false : ignoreUnlabeled); + }); + static { - PARSER.declareInt(PrecisionAtK::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); - PARSER.declareBoolean(PrecisionAtK::setIgnoreUnlabeled, IGNORE_UNLABELED_FIELD); + PARSER.declareInt(optionalConstructorArg(), RELEVANT_RATING_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), IGNORE_UNLABELED_FIELD); } - public PrecisionAtK(StreamInput in) throws IOException { - relevantRatingThreshhold = in.readOptionalVInt(); - ignoreUnlabeled = in.readOptionalBoolean(); + PrecisionAtK(StreamInput in) throws IOException { + relevantRatingThreshhold = in.readVInt(); + ignoreUnlabeled = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeOptionalVInt(relevantRatingThreshhold); - out.writeOptionalBoolean(ignoreUnlabeled); + out.writeVInt(relevantRatingThreshhold); + out.writeBoolean(ignoreUnlabeled); } @Override @@ -88,18 +104,6 @@ public class PrecisionAtK implements RankedListQualityMetric { return NAME; } - /** - * Sets the rating threshold above which ratings are considered to be - * "relevant" for this metric. - */ - public void setRelevantRatingThreshhold(int threshold) { - if (threshold < 0) { - throw new IllegalArgumentException( - "Relevant rating threshold for precision must be positive integer."); - } - this.relevantRatingThreshhold = threshold; - } - /** * Return the rating threshold above which ratings are considered to be * "relevant" for this metric. Defaults to 1. @@ -108,13 +112,6 @@ public class PrecisionAtK implements RankedListQualityMetric { return relevantRatingThreshhold; } - /** - * Sets the 'ìgnore_unlabeled' parameter - */ - public void setIgnoreUnlabeled(boolean ignoreUnlabeled) { - this.ignoreUnlabeled = ignoreUnlabeled; - } - /** * Gets the 'ìgnore_unlabeled' parameter */ diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index 69825405363..189f7ab91ac 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -56,24 +56,19 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin { } /** - * Returns parsers for {@link NamedWriteable} this plugin will use over the - * transport protocol. - * + * Returns parsers for {@link NamedWriteable} objects that this plugin sends over the transport protocol. * @see NamedWriteableRegistry */ @Override public List getNamedWriteables() { List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, - PrecisionAtK.NAME, PrecisionAtK::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, - MeanReciprocalRank.NAME, MeanReciprocalRank::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, - DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, PrecisionAtK.NAME, - PrecisionAtK.Breakdown::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, - MeanReciprocalRank.NAME, MeanReciprocalRank.Breakdown::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetric.class, PrecisionAtK.NAME, PrecisionAtK::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetric.class, MeanReciprocalRank.NAME, MeanReciprocalRank::new)); + namedWriteables.add( + new NamedWriteableRegistry.Entry(EvaluationMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, PrecisionAtK.NAME, PrecisionAtK.Breakdown::new)); + namedWriteables + .add(new NamedWriteableRegistry.Entry(MetricDetails.class, MeanReciprocalRank.NAME, MeanReciprocalRank.Breakdown::new)); return namedWriteables; } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 60d0a5dd4fd..3a88bc4de57 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -34,7 +34,9 @@ import org.elasticsearch.script.Script; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -53,9 +55,9 @@ public class RankEvalSpec implements Writeable, ToXContentObject { * Collection of query specifications, that is e.g. search request templates * to use for query translation. */ - private Collection ratedRequests = new ArrayList<>(); + private final List ratedRequests; /** Definition of the quality metric, e.g. precision at N */ - private RankedListQualityMetric metric; + private final EvaluationMetric metric; /** Maximum number of requests to execute in parallel. */ private int maxConcurrentSearches = MAX_CONCURRENT_SEARCHES; /** Default max number of requests. */ @@ -63,7 +65,7 @@ public class RankEvalSpec implements Writeable, ToXContentObject { /** optional: Templates to base test requests on */ private Map templates = new HashMap<>(); - public RankEvalSpec(Collection ratedRequests, RankedListQualityMetric metric, + public RankEvalSpec(List ratedRequests, EvaluationMetric metric, Collection templates) { if (ratedRequests == null || ratedRequests.size() < 1) { throw new IllegalStateException( @@ -92,7 +94,7 @@ public class RankEvalSpec implements Writeable, ToXContentObject { } } - public RankEvalSpec(Collection ratedRequests, RankedListQualityMetric metric) { + public RankEvalSpec(List ratedRequests, EvaluationMetric metric) { this(ratedRequests, metric, null); } @@ -102,7 +104,7 @@ public class RankEvalSpec implements Writeable, ToXContentObject { for (int i = 0; i < specSize; i++) { ratedRequests.add(new RatedRequest(in)); } - metric = in.readNamedWriteable(RankedListQualityMetric.class); + metric = in.readNamedWriteable(EvaluationMetric.class); int size = in.readVInt(); for (int i = 0; i < size; i++) { String key = in.readString(); @@ -128,13 +130,13 @@ public class RankEvalSpec implements Writeable, ToXContentObject { } /** Returns the metric to use for quality evaluation.*/ - public RankedListQualityMetric getMetric() { + public EvaluationMetric getMetric() { return metric; } /** Returns a list of intent to query translation specifications to evaluate. */ - public Collection getRatedRequests() { - return ratedRequests; + public List getRatedRequests() { + return Collections.unmodifiableList(ratedRequests); } /** Returns the template to base test requests on. */ @@ -160,8 +162,8 @@ public class RankEvalSpec implements Writeable, ToXContentObject { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rank_eval", - a -> new RankEvalSpec((Collection) a[0], - (RankedListQualityMetric) a[1], (Collection) a[2])); + a -> new RankEvalSpec((List) a[0], + (EvaluationMetric) a[1], (Collection) a[2])); static { PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> { @@ -169,7 +171,7 @@ public class RankEvalSpec implements Writeable, ToXContentObject { } , REQUESTS_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { try { - return RankedListQualityMetric.fromXContent(p); + return EvaluationMetric.fromXContent(p); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index ec19aa8aa0e..fc5880e60cd 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -33,13 +33,24 @@ import java.io.IOException; import java.util.Objects; /** - * A document ID and its rating for the query QA use case. + * Represents a document (specified by its _index/_id) and its corresponding rating + * with respect to a specific search query. + *

+ * Json structure in a request: + * 

+ * {
+ *   "_index": "my_index",
+ *   "_id": "doc1",
+ *   "rating": 0
+ * }
+ *
+ * */ public class RatedDocument implements Writeable, ToXContentObject { - public static final ParseField RATING_FIELD = new ParseField("rating"); - public static final ParseField DOC_ID_FIELD = new ParseField("_id"); - public static final ParseField INDEX_FIELD = new ParseField("_index"); + static final ParseField RATING_FIELD = new ParseField("rating"); + static final ParseField DOC_ID_FIELD = new ParseField("_id"); + static final ParseField INDEX_FIELD = new ParseField("_index"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rated_document", a -> new RatedDocument((String) a[0], (String) a[1], (Integer) a[2])); @@ -50,23 +61,19 @@ public class RatedDocument implements Writeable, ToXContentObject { PARSER.declareInt(ConstructingObjectParser.constructorArg(), RATING_FIELD); } - private int rating; - private DocumentKey key; + private final int rating; + private final DocumentKey key; - public RatedDocument(String index, String docId, int rating) { - this(new DocumentKey(index, docId), rating); - } - - public RatedDocument(StreamInput in) throws IOException { - this.key = new DocumentKey(in); - this.rating = in.readVInt(); - } - - public RatedDocument(DocumentKey ratedDocumentKey, int rating) { - this.key = ratedDocumentKey; + public RatedDocument(String index, String id, int rating) { + this.key = new DocumentKey(index, id); this.rating = rating; } + RatedDocument(StreamInput in) throws IOException { + this.key = new DocumentKey(in.readString(), in.readString()); + this.rating = in.readVInt(); + } + public DocumentKey getKey() { return this.key; } @@ -76,7 +83,7 @@ public class RatedDocument implements Writeable, ToXContentObject { } public String getDocID() { - return key.getDocID(); + return key.getDocId(); } public int getRating() { @@ -85,11 +92,12 @@ public class RatedDocument implements Writeable, ToXContentObject { @Override public void writeTo(StreamOutput out) throws IOException { - this.key.writeTo(out); + out.writeString(key.getIndex()); + out.writeString(key.getDocId()); out.writeVInt(rating); } - public static RatedDocument fromXContent(XContentParser parser) { + static RatedDocument fromXContent(XContentParser parser) { return PARSER.apply(parser, null); } @@ -97,7 +105,7 @@ public class RatedDocument implements Writeable, ToXContentObject { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(INDEX_FIELD.getPreferredName(), key.getIndex()); - builder.field(DOC_ID_FIELD.getPreferredName(), key.getDocID()); + builder.field(DOC_ID_FIELD.getPreferredName(), key.getDocId()); builder.field(RATING_FIELD.getPreferredName(), rating); builder.endObject(); return builder; @@ -124,4 +132,55 @@ public class RatedDocument implements Writeable, ToXContentObject { public final int hashCode() { return Objects.hash(key, rating); } + + /** + * a joint document key consisting of the documents index and id + */ + static class DocumentKey { + + private final String docId; + private final String index; + + DocumentKey(String index, String docId) { + if (Strings.isNullOrEmpty(index)) { + throw new IllegalArgumentException("Index must be set for each rated document"); + } + if (Strings.isNullOrEmpty(docId)) { + throw new IllegalArgumentException("DocId must be set for each rated document"); + } + + this.index = index; + this.docId = docId; + } + + String getIndex() { + return index; + } + + String getDocId() { + return docId; + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + DocumentKey other = (DocumentKey) obj; + return Objects.equals(index, other.index) && Objects.equals(docId, other.docId); + } + + @Override + public final int hashCode() { + return Objects.hash(index, docId); + } + + @Override + public String toString() { + return "{\"_index\":\"" + index + "\",\"_id\":\"" + docId + "\"}"; + } + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index af322add5ec..e8ed925987f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 8c9024d0500..4e3fa8afa46 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -157,11 +157,11 @@ public class TransportRankEvalAction private RatedRequest specification; private Map requestDetails; private Map errors; - private RankedListQualityMetric metric; + private EvaluationMetric metric; private AtomicInteger responseCounter; public RankEvalActionListener(ActionListener listener, - RankedListQualityMetric metric, RatedRequest specification, + EvaluationMetric metric, RatedRequest specification, Map details, Map errors, AtomicInteger responseCounter) { this.listener = listener; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 4154000a7b0..b6bd80f497e 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -36,20 +37,23 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; +import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments; +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; + public class DiscountedCumulativeGainTests extends ESTestCase { /** * Assuming the docs are ranked in the following order: * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / - * log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) * ------------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 - * 3 | 3 | 7.0 | 2.0 | 3.5 4 | 0 | 0.0 | 2.321928094887362 | 0.0 5 | 1 | 1.0 - * | 2.584962500721156 | 0.38685280723454163 6 | 2 | 3.0 | 2.807354922057604 - * | 1.0686215613240666 + * 1 | 3 | 7.0 | 1.0 | 7.0 2 |  + * 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 + * 3 | 3 | 7.0 | 2.0 | 3.5 + * 4 | 0 | 0.0 | 2.321928094887362 | 0.0 + * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163 + * 6 | 2 | 3.0 | 2.807354922057604 | 1.0686215613240666 * * dcg = 13.84826362927298 (sum of last column) */ @@ -69,17 +73,18 @@ public class DiscountedCumulativeGainTests extends ESTestCase { * Check with normalization: to get the maximal possible dcg, sort documents by * relevance in descending order * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / - * log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) * --------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0  | 7.0 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 - * 3 | 2 | 3.0 | 2.0  | 1.5 4 | 2 | 3.0 | 2.321928094887362  - * | 1.2920296742201793 5 | 1 | 1.0 | 2.584962500721156  | 0.38685280723454163 6 - * | 0 | 0.0 | 2.807354922057604  | 0.0 + * 1 | 3 | 7.0 | 1.0  | 7.0 + * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 + * 3 | 2 | 3.0 | 2.0  | 1.5 + * 4 | 2 | 3.0 | 2.321928094887362 | 1.2920296742201793 + * 5 | 1 | 1.0 | 2.584962500721156  | 0.38685280723454163 + * 6 | 0 | 0.0 | 2.807354922057604  | 0.0 * * idcg = 14.595390756454922 (sum of last column) */ - dcg.setNormalize(true); + dcg = new DiscountedCumulativeGain(true, null); assertEquals(13.84826362927298 / 14.595390756454922, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); } @@ -87,12 +92,14 @@ public class DiscountedCumulativeGainTests extends ESTestCase { * This tests metric when some documents in the search result don't have a * rating provided by the user. * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / - * log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) * ------------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 - * 3 | 3 | 7.0 | 2.0 | 3.5 4 | n/a | n/a | n/a | n/a 5 | 1 | 1.0 - * | 2.584962500721156 | 0.38685280723454163 6 | n/a | n/a | n/a | n/a + * 1 | 3 | 7.0 | 1.0 | 7.0 2 |  + * 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 + * 3 | 3 | 7.0 | 2.0 | 3.5 + * 4 | n/a | n/a | n/a | n/a + * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163 + * 6 | n/a | n/a | n/a | n/a * * dcg = 12.779642067948913 (sum of last column) */ @@ -118,16 +125,18 @@ public class DiscountedCumulativeGainTests extends ESTestCase { * Check with normalization: to get the maximal possible dcg, sort documents by * relevance in descending order * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / - * log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) * ---------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0  | 7.0 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 - * 3 | 2 | 3.0 | 2.0  | 1.5 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339 - * 5 | n.a | n.a | n.a.  | n.a. 6 | n.a | n.a | n.a  | n.a + * 1 | 3 | 7.0 | 1.0  | 7.0 + * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 + * 3 | 2 | 3.0 | 2.0  | 1.5 + * 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339 + * 5 | n.a | n.a | n.a.  | n.a. + * 6 | n.a | n.a | n.a  | n.a * * idcg = 13.347184833073591 (sum of last column) */ - dcg.setNormalize(true); + dcg = new DiscountedCumulativeGain(true, null); assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); } @@ -136,13 +145,15 @@ public class DiscountedCumulativeGainTests extends ESTestCase { * documents than search hits because we restrict DCG to be calculated at the * fourth position * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / - * log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) * ------------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 - * 3 | 3 | 7.0 | 2.0 | 3.5 4 | n/a | n/a | n/a | n/a - * ----------------------------------------------------------------- 5 | 1 | 1.0 - * | 2.584962500721156 | 0.38685280723454163 6 | n/a | n/a | n/a | n/a + * 1 | 3 | 7.0 | 1.0 | 7.0 2 |  + * 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 + * 3 | 3 | 7.0 | 2.0 | 3.5 + * 4 | n/a | n/a | n/a | n/a + * ----------------------------------------------------------------- + * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163 + * 6 | n/a | n/a | n/a | n/a * * dcg = 12.392789260714371 (sum of last column until position 4) */ @@ -171,22 +182,24 @@ public class DiscountedCumulativeGainTests extends ESTestCase { * Check with normalization: to get the maximal possible dcg, sort documents by * relevance in descending order * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / - * log_2(rank + 1) + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) * --------------------------------------------------------------------------------------- - * 1 | 3 | 7.0 | 1.0  | 7.0 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 - * 3 | 2 | 3.0 | 2.0  | 1.5 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339 + * 1 | 3 | 7.0 | 1.0  | 7.0 + * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 + * 3 | 2 | 3.0 | 2.0  | 1.5 + * 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339 * --------------------------------------------------------------------------------------- - * 5 | n.a | n.a | n.a.  | n.a. 6 | n.a | n.a | n.a  | n.a + * 5 | n.a | n.a | n.a.  | n.a. + * 6 | n.a | n.a | n.a  | n.a * * idcg = 13.347184833073591 (sum of last column) */ - dcg.setNormalize(true); + dcg = new DiscountedCumulativeGain(true, null); assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), 0.00001); } public void testParseFromXContent() throws IOException { - String xContent = " {\n" + " \"unknown_doc_rating\": 2,\n" + " \"normalize\": true\n" + "}"; + String xContent = " { \"unknown_doc_rating\": 2, \"normalize\": true }"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) { DiscountedCumulativeGain dcgAt = DiscountedCumulativeGain.fromXContent(parser); assertEquals(2, dcgAt.getUnknownDocRating().intValue()); @@ -217,29 +230,25 @@ public class DiscountedCumulativeGainTests extends ESTestCase { public void testSerialization() throws IOException { DiscountedCumulativeGain original = createTestItem(); - DiscountedCumulativeGain deserialized = RankEvalTestHelper.copy(original, DiscountedCumulativeGain::new); + DiscountedCumulativeGain deserialized = ESTestCase.copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()), + DiscountedCumulativeGain::new); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); } public void testEqualsAndHash() throws IOException { - DiscountedCumulativeGain testItem = createTestItem(); - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - RankEvalTestHelper.copy(testItem, DiscountedCumulativeGain::new)); + checkEqualsAndHashCode(createTestItem(), original -> { + return new DiscountedCumulativeGain(original.getNormalize(), original.getUnknownDocRating()); + }, DiscountedCumulativeGainTests::mutateTestItem); } private static DiscountedCumulativeGain mutateTestItem(DiscountedCumulativeGain original) { - boolean normalise = original.getNormalize(); - int unknownDocRating = original.getUnknownDocRating(); - DiscountedCumulativeGain gain = new DiscountedCumulativeGain(); - gain.setNormalize(normalise); - gain.setUnknownDocRating(unknownDocRating); - - List mutators = new ArrayList<>(); - mutators.add(() -> gain.setNormalize(!original.getNormalize())); - mutators.add(() -> gain.setUnknownDocRating(randomValueOtherThan(unknownDocRating, () -> randomIntBetween(0, 10)))); - randomFrom(mutators).run(); - return gain; + if (randomBoolean()) { + return new DiscountedCumulativeGain(!original.getNormalize(), original.getUnknownDocRating()); + } else { + return new DiscountedCumulativeGain(original.getNormalize(), + randomValueOtherThan(original.getUnknownDocRating(), () -> randomIntBetween(0, 10))); + } } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java deleted file mode 100644 index 7a241451bc9..00000000000 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DocumentKeyTests.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.rankeval; - -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; - -public class DocumentKeyTests extends ESTestCase { - - static DocumentKey createRandomRatedDocumentKey() { - String index = randomAlphaOfLengthBetween(1, 10); - String docId = randomAlphaOfLengthBetween(1, 10); - return new DocumentKey(index, docId); - } - - public DocumentKey createTestItem() { - return createRandomRatedDocumentKey(); - } - - public DocumentKey mutateTestItem(DocumentKey original) { - String index = original.getIndex(); - String docId = original.getDocID(); - switch (randomIntBetween(0, 1)) { - case 0: - index = index + "_"; - break; - case 1: - docId = docId + "_"; - break; - default: - throw new IllegalStateException("The test should only allow two parameters mutated"); - } - return new DocumentKey(index, docId); - } - - public void testEqualsAndHash() throws IOException { - DocumentKey testItem = createRandomRatedDocumentKey(); - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - new DocumentKey(testItem.getIndex(), testItem.getDocID())); - } - - public void testSerialization() throws IOException { - DocumentKey original = createTestItem(); - DocumentKey deserialized = RankEvalTestHelper.copy(original, DocumentKey::new); - assertEquals(deserialized, original); - assertEquals(deserialized.hashCode(), original.hashCode()); - assertNotSame(deserialized, original); - } -} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java index 97e40f12f30..fb1c7db554a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -20,22 +20,24 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey; import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; + public class EvalQueryQualityTests extends ESTestCase { - private static NamedWriteableRegistry namedWritableRegistry = new NamedWriteableRegistry( - new RankEvalPlugin().getNamedWriteables()); + private static NamedWriteableRegistry namedWritableRegistry = new NamedWriteableRegistry(new RankEvalPlugin().getNamedWriteables()); public static EvalQueryQuality randomEvalQueryQuality() { List unknownDocs = new ArrayList<>(); int numberOfUnknownDocs = randomInt(5); for (int i = 0; i < numberOfUnknownDocs; i++) { - unknownDocs.add(DocumentKeyTests.createRandomRatedDocumentKey()); + unknownDocs.add(new DocumentKey(randomAlphaOfLength(10), randomAlphaOfLength(10))); } int numberOfSearchHits = randomInt(5); List ratedHits = new ArrayList<>(); @@ -54,17 +56,18 @@ public class EvalQueryQualityTests extends ESTestCase { public void testSerialization() throws IOException { EvalQueryQuality original = randomEvalQueryQuality(); - EvalQueryQuality deserialized = RankEvalTestHelper.copy(original, EvalQueryQuality::new, - namedWritableRegistry); + EvalQueryQuality deserialized = copy(original); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); } + private static EvalQueryQuality copy(EvalQueryQuality original) throws IOException { + return ESTestCase.copyWriteable(original, namedWritableRegistry, EvalQueryQuality::new); + } + public void testEqualsAndHash() throws IOException { - EvalQueryQuality testItem = randomEvalQueryQuality(); - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - RankEvalTestHelper.copy(testItem, EvalQueryQuality::new, namedWritableRegistry)); + checkEqualsAndHashCode(randomEvalQueryQuality(), EvalQueryQualityTests::copy, EvalQueryQualityTests::mutateTestItem); } private static EvalQueryQuality mutateTestItem(EvalQueryQuality original) { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java similarity index 79% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java index bed24a18505..81586783c06 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java @@ -19,14 +19,15 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.Index; -import org.elasticsearch.index.rankeval.PrecisionTests.Rating; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.test.ESTestCase; @@ -38,21 +39,35 @@ import java.util.Collections; import java.util.List; import java.util.Vector; -public class ReciprocalRankTests extends ESTestCase { +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; + +public class MeanReciprocalRankTests extends ESTestCase { + + public void testParseFromXContent() throws IOException { + String xContent = "{ }"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) { + MeanReciprocalRank mrr = MeanReciprocalRank.fromXContent(parser); + assertEquals(1, mrr.getRelevantRatingThreshold()); + } + + xContent = "{ \"relevant_rating_threshold\": 2 }"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) { + MeanReciprocalRank mrr = MeanReciprocalRank.fromXContent(parser); + assertEquals(2, mrr.getRelevantRatingThreshold()); + } + } public void testMaxAcceptableRank() { MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(); - int searchHits = randomIntBetween(1, 50); - SearchHit[] hits = createSearchHits(0, searchHits, "test"); List ratedDocs = new ArrayList<>(); int relevantAt = randomIntBetween(0, searchHits); for (int i = 0; i <= searchHits; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", Integer.toString(i), TestRatingEnum.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", Integer.toString(i), TestRatingEnum.IRRELEVANT.ordinal())); } } @@ -76,9 +91,9 @@ public class ReciprocalRankTests extends ESTestCase { int relevantAt = randomIntBetween(0, 9); for (int i = 0; i <= 20; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", Integer.toString(i), TestRatingEnum.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument("test", Integer.toString(i), TestRatingEnum.IRRELEVANT.ordinal())); } } @@ -101,8 +116,7 @@ public class ReciprocalRankTests extends ESTestCase { rated.add(new RatedDocument("test", "4", 4)); SearchHit[] hits = createSearchHits(0, 5, "test"); - MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(); - reciprocalRank.setRelevantRatingThreshhold(2); + MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(2); EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, rated); assertEquals((double) 1 / 3, evaluation.getQualityLevel(), 0.00001); assertEquals(3, ((MeanReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); @@ -153,35 +167,31 @@ public class ReciprocalRankTests extends ESTestCase { } private static MeanReciprocalRank createTestItem() { - MeanReciprocalRank testItem = new MeanReciprocalRank(); - testItem.setRelevantRatingThreshhold(randomIntBetween(0, 20)); - return testItem; + return new MeanReciprocalRank(randomIntBetween(0, 20)); } public void testSerialization() throws IOException { MeanReciprocalRank original = createTestItem(); - - MeanReciprocalRank deserialized = RankEvalTestHelper.copy(original, MeanReciprocalRank::new); + MeanReciprocalRank deserialized = ESTestCase.copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()), + MeanReciprocalRank::new); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); } public void testEqualsAndHash() throws IOException { - MeanReciprocalRank testItem = createTestItem(); - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - RankEvalTestHelper.copy(testItem, MeanReciprocalRank::new)); + checkEqualsAndHashCode(createTestItem(), MeanReciprocalRankTests::copy, MeanReciprocalRankTests::mutate); } - private static MeanReciprocalRank mutateTestItem(MeanReciprocalRank testItem) { - int relevantThreshold = testItem.getRelevantRatingThreshold(); - MeanReciprocalRank rank = new MeanReciprocalRank(); - rank.setRelevantRatingThreshhold(randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10))); - return rank; + private static MeanReciprocalRank copy(MeanReciprocalRank testItem) { + return new MeanReciprocalRank(testItem.getRelevantRatingThreshold()); + } + + private static MeanReciprocalRank mutate(MeanReciprocalRank testItem) { + return new MeanReciprocalRank(randomValueOtherThan(testItem.getRelevantRatingThreshold(), () -> randomIntBetween(0, 10))); } public void testInvalidRelevantThreshold() { - MeanReciprocalRank prez = new MeanReciprocalRank(); - expectThrows(IllegalArgumentException.class, () -> prez.setRelevantRatingThreshhold(-1)); + expectThrows(IllegalArgumentException.class, () -> new MeanReciprocalRank(-1)); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java similarity index 74% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java index d520174c28e..01ea5e7cf65 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -38,11 +39,13 @@ import java.util.Collections; import java.util.List; import java.util.Vector; -public class PrecisionTests extends ESTestCase { +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; + +public class PrecisionAtKTests extends ESTestCase { public void testPrecisionAtFiveCalculation() { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal())); + rated.add(createRatedDoc("test", "0", TestRatingEnum.RELEVANT.ordinal())); EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated, "test"), rated); assertEquals(1, evaluated.getQualityLevel(), 0.00001); assertEquals(1, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); @@ -51,11 +54,11 @@ public class PrecisionTests extends ESTestCase { public void testPrecisionAtFiveIgnoreOneResult() { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "2", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "3", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "4", Rating.IRRELEVANT.ordinal())); + rated.add(createRatedDoc("test", "0", TestRatingEnum.RELEVANT.ordinal())); + rated.add(createRatedDoc("test", "1", TestRatingEnum.RELEVANT.ordinal())); + rated.add(createRatedDoc("test", "2", TestRatingEnum.RELEVANT.ordinal())); + rated.add(createRatedDoc("test", "3", TestRatingEnum.RELEVANT.ordinal())); + rated.add(createRatedDoc("test", "4", TestRatingEnum.IRRELEVANT.ordinal())); EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated, "test"), rated); assertEquals((double) 4 / 5, evaluated.getQualityLevel(), 0.00001); assertEquals(4, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); @@ -69,13 +72,12 @@ public class PrecisionTests extends ESTestCase { */ public void testPrecisionAtFiveRelevanceThreshold() { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "0", 0)); - rated.add(new RatedDocument("test", "1", 1)); - rated.add(new RatedDocument("test", "2", 2)); - rated.add(new RatedDocument("test", "3", 3)); - rated.add(new RatedDocument("test", "4", 4)); - PrecisionAtK precisionAtN = new PrecisionAtK(); - precisionAtN.setRelevantRatingThreshhold(2); + rated.add(createRatedDoc("test", "0", 0)); + rated.add(createRatedDoc("test", "1", 1)); + rated.add(createRatedDoc("test", "2", 2)); + rated.add(createRatedDoc("test", "3", 3)); + rated.add(createRatedDoc("test", "4", 4)); + PrecisionAtK precisionAtN = new PrecisionAtK(2, false); EvalQueryQuality evaluated = precisionAtN.evaluate("id", toSearchHits(rated, "test"), rated); assertEquals((double) 3 / 5, evaluated.getQualityLevel(), 0.00001); assertEquals(3, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); @@ -84,11 +86,11 @@ public class PrecisionTests extends ESTestCase { public void testPrecisionAtFiveCorrectIndex() { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test_other", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test_other", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "2", Rating.IRRELEVANT.ordinal())); + rated.add(createRatedDoc("test_other", "0", TestRatingEnum.RELEVANT.ordinal())); + rated.add(createRatedDoc("test_other", "1", TestRatingEnum.RELEVANT.ordinal())); + rated.add(createRatedDoc("test", "0", TestRatingEnum.RELEVANT.ordinal())); + rated.add(createRatedDoc("test", "1", TestRatingEnum.RELEVANT.ordinal())); + rated.add(createRatedDoc("test", "2", TestRatingEnum.IRRELEVANT.ordinal())); // the following search hits contain only the last three documents EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated.subList(2, 5), "test"), rated); assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); @@ -98,8 +100,8 @@ public class PrecisionTests extends ESTestCase { public void testIgnoreUnlabeled() { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "1", Rating.RELEVANT.ordinal())); + rated.add(createRatedDoc("test", "0", TestRatingEnum.RELEVANT.ordinal())); + rated.add(createRatedDoc("test", "1", TestRatingEnum.RELEVANT.ordinal())); // add an unlabeled search hit SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test"), 3); searchHits[2] = new SearchHit(2, "2", new Text("testtype"), Collections.emptyMap()); @@ -111,8 +113,7 @@ public class PrecisionTests extends ESTestCase { assertEquals(3, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); // also try with setting `ignore_unlabeled` - PrecisionAtK prec = new PrecisionAtK(); - prec.setIgnoreUnlabeled(true); + PrecisionAtK prec = new PrecisionAtK(1, true); evaluated = prec.evaluate("id", searchHits, rated); assertEquals((double) 2 / 2, evaluated.getQualityLevel(), 0.00001); assertEquals(2, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); @@ -131,8 +132,7 @@ public class PrecisionTests extends ESTestCase { assertEquals(5, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); // also try with setting `ignore_unlabeled` - PrecisionAtK prec = new PrecisionAtK(); - prec.setIgnoreUnlabeled(true); + PrecisionAtK prec = new PrecisionAtK(1, true); evaluated = prec.evaluate("id", hits, Collections.emptyList()); assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); @@ -158,16 +158,11 @@ public class PrecisionTests extends ESTestCase { public void testInvalidRelevantThreshold() { PrecisionAtK prez = new PrecisionAtK(); - expectThrows(IllegalArgumentException.class, () -> prez.setRelevantRatingThreshhold(-1)); + expectThrows(IllegalArgumentException.class, () -> new PrecisionAtK(-1, false)); } public static PrecisionAtK createTestItem() { - PrecisionAtK precision = new PrecisionAtK(); - if (randomBoolean()) { - precision.setRelevantRatingThreshhold(randomIntBetween(0, 10)); - } - precision.setIgnoreUnlabeled(randomBoolean()); - return precision; + return new PrecisionAtK(randomIntBetween(0, 10), randomBoolean()); } public void testXContentRoundtrip() throws IOException { @@ -186,29 +181,28 @@ public class PrecisionTests extends ESTestCase { public void testSerialization() throws IOException { PrecisionAtK original = createTestItem(); - PrecisionAtK deserialized = RankEvalTestHelper.copy(original, PrecisionAtK::new); + PrecisionAtK deserialized = ESTestCase.copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()), + PrecisionAtK::new); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); } public void testEqualsAndHash() throws IOException { - PrecisionAtK testItem = createTestItem(); - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), RankEvalTestHelper.copy(testItem, PrecisionAtK::new)); + checkEqualsAndHashCode(createTestItem(), PrecisionAtKTests::copy, PrecisionAtKTests::mutate); } - private static PrecisionAtK mutateTestItem(PrecisionAtK original) { - boolean ignoreUnlabeled = original.getIgnoreUnlabeled(); - int relevantThreshold = original.getRelevantRatingThreshold(); - PrecisionAtK precision = new PrecisionAtK(); - precision.setIgnoreUnlabeled(ignoreUnlabeled); - precision.setRelevantRatingThreshhold(relevantThreshold); + private static PrecisionAtK copy(PrecisionAtK original) { + return new PrecisionAtK(original.getRelevantRatingThreshold(), original.getIgnoreUnlabeled()); + } - List mutators = new ArrayList<>(); - mutators.add(() -> precision.setIgnoreUnlabeled(!ignoreUnlabeled)); - mutators.add(() -> precision.setRelevantRatingThreshhold(randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10)))); - randomFrom(mutators).run(); - return precision; + private static PrecisionAtK mutate(PrecisionAtK original) { + if (randomBoolean()) { + return new PrecisionAtK(original.getRelevantRatingThreshold(), !original.getIgnoreUnlabeled()); + } else { + return new PrecisionAtK(randomValueOtherThan(original.getRelevantRatingThreshold(), () -> randomIntBetween(0, 10)), + original.getIgnoreUnlabeled()); + } } private static SearchHit[] toSearchHits(List rated, String index) { @@ -220,7 +214,7 @@ public class PrecisionTests extends ESTestCase { return hits; } - public enum Rating { - IRRELEVANT, RELEVANT; + private static RatedDocument createRatedDoc(String index, String id, int rating) { + return new RatedDocument(index, id, rating); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 2e3f86543c6..3b674d7e8a6 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -22,7 +22,6 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.rankeval.PrecisionTests.Rating; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; @@ -35,7 +34,7 @@ import java.util.List; import java.util.Map.Entry; import java.util.Set; -import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments; +import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments; public class RankEvalRequestIT extends ESIntegTestCase { @Override @@ -82,8 +81,7 @@ public class RankEvalRequestIT extends ESIntegTestCase { specifications.add(berlinRequest); - PrecisionAtK metric = new PrecisionAtK(); - metric.setIgnoreUnlabeled(true); + PrecisionAtK metric = new PrecisionAtK(1, true); RankEvalSpec task = new RankEvalSpec(specifications, metric); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), @@ -106,7 +104,7 @@ public class RankEvalRequestIT extends ESIntegTestCase { if (id.equals("1") || id.equals("6")) { assertFalse(hit.getRating().isPresent()); } else { - assertEquals(Rating.RELEVANT.ordinal(), hit.getRating().get().intValue()); + assertEquals(TestRatingEnum.RELEVANT.ordinal(), hit.getRating().get().intValue()); } } } @@ -117,7 +115,7 @@ public class RankEvalRequestIT extends ESIntegTestCase { for (RatedSearchHit hit : hitsAndRatings) { String id = hit.getSearchHit().getId(); if (id.equals("1")) { - assertEquals(Rating.RELEVANT.ordinal(), hit.getRating().get().intValue()); + assertEquals(TestRatingEnum.RELEVANT.ordinal(), hit.getRating().get().intValue()); } else { assertFalse(hit.getRating().isPresent()); } @@ -167,7 +165,7 @@ public class RankEvalRequestIT extends ESIntegTestCase { private static List createRelevant(String... docs) { List relevant = new ArrayList<>(); for (String doc : docs) { - relevant.add(new RatedDocument("test", doc, Rating.RELEVANT.ordinal())); + relevant.add(new RatedDocument("test", doc, TestRatingEnum.RELEVANT.ordinal())); } return relevant; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index 497f17de1b9..aff758c096b 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -43,7 +44,7 @@ public class RankEvalResponseTests extends ESTestCase { int numberOfUnknownDocs = randomIntBetween(0, 5); List unknownDocs = new ArrayList<>(numberOfUnknownDocs); for (int d = 0; d < numberOfUnknownDocs; d++) { - unknownDocs.add(DocumentKeyTests.createRandomRatedDocumentKey()); + unknownDocs.add(new DocumentKey(randomAlphaOfLength(10), randomAlphaOfLength(10))); } EvalQueryQuality evalQuality = new EvalQueryQuality(id, randomDoubleBetween(0.0, 1.0, true)); @@ -65,12 +66,9 @@ public class RankEvalResponseTests extends ESTestCase { try (StreamInput in = output.bytes().streamInput()) { RankEvalResponse deserializedResponse = new RankEvalResponse(); deserializedResponse.readFrom(in); - assertEquals(randomResponse.getQualityLevel(), - deserializedResponse.getQualityLevel(), Double.MIN_VALUE); - assertEquals(randomResponse.getPartialResults(), - deserializedResponse.getPartialResults()); - assertEquals(randomResponse.getFailures().keySet(), - deserializedResponse.getFailures().keySet()); + assertEquals(randomResponse.getQualityLevel(), deserializedResponse.getQualityLevel(), Double.MIN_VALUE); + assertEquals(randomResponse.getPartialResults(), deserializedResponse.getPartialResults()); + assertEquals(randomResponse.getFailures().keySet(), deserializedResponse.getFailures().keySet()); assertNotSame(randomResponse, deserializedResponse); assertEquals(-1, in.read()); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 40624f12dab..6197efb1b04 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -45,6 +45,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.function.Supplier; +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; + public class RankEvalSpecTests extends ESTestCase { private static List randomList(Supplier randomSupplier) { @@ -57,9 +59,9 @@ public class RankEvalSpecTests extends ESTestCase { } private static RankEvalSpec createTestItem() throws IOException { - RankedListQualityMetric metric; + EvaluationMetric metric; if (randomBoolean()) { - metric = PrecisionTests.createTestItem(); + metric = PrecisionAtKTests.createTestItem(); } else { metric = DiscountedCumulativeGainTests.createTestItem(); } @@ -111,41 +113,30 @@ public class RankEvalSpecTests extends ESTestCase { public void testSerialization() throws IOException { RankEvalSpec original = createTestItem(); - - List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, PrecisionAtK.NAME, PrecisionAtK::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, - DiscountedCumulativeGain::new)); - namedWriteables - .add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, MeanReciprocalRank.NAME, MeanReciprocalRank::new)); - - RankEvalSpec deserialized = RankEvalTestHelper.copy(original, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables)); + RankEvalSpec deserialized = copy(original); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); } - public void testEqualsAndHash() throws IOException { - RankEvalSpec testItem = createTestItem(); - + private static RankEvalSpec copy(RankEvalSpec original) throws IOException { List namedWriteables = new ArrayList<>(); namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, PrecisionAtK.NAME, PrecisionAtK::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGain.NAME, - DiscountedCumulativeGain::new)); - namedWriteables - .add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, MeanReciprocalRank.NAME, MeanReciprocalRank::new)); - - RankEvalSpec mutant = RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables)); - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(mutant), - RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables))); + namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetric.class, PrecisionAtK.NAME, PrecisionAtK::new)); + namedWriteables.add( + new NamedWriteableRegistry.Entry(EvaluationMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetric.class, MeanReciprocalRank.NAME, MeanReciprocalRank::new)); + return ESTestCase.copyWriteable(original, new NamedWriteableRegistry(namedWriteables), RankEvalSpec::new); } - private static RankEvalSpec mutateTestItem(RankEvalSpec mutant) { - Collection ratedRequests = mutant.getRatedRequests(); - RankedListQualityMetric metric = mutant.getMetric(); - Map templates = mutant.getTemplates(); + public void testEqualsAndHash() throws IOException { + checkEqualsAndHashCode(createTestItem(), RankEvalSpecTests::copy, RankEvalSpecTests::mutateTestItem); + } + + private static RankEvalSpec mutateTestItem(RankEvalSpec original) { + List ratedRequests = new ArrayList<>(original.getRatedRequests()); + EvaluationMetric metric = original.getMetric(); + Map templates = original.getTemplates(); int mutate = randomIntBetween(0, 2); switch (mutate) { @@ -177,7 +168,7 @@ public class RankEvalSpecTests extends ESTestCase { } public void testMissingRatedRequestsFailsParsing() { - RankedListQualityMetric metric = new PrecisionAtK(); + EvaluationMetric metric = new PrecisionAtK(); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(new ArrayList<>(), metric)); expectThrows(IllegalStateException.class, () -> new RankEvalSpec(null, metric)); } @@ -189,7 +180,7 @@ public class RankEvalSpecTests extends ESTestCase { } public void testMissingTemplateAndSearchRequestFailsParsing() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1)); Map params = new HashMap<>(); params.put("key", "value"); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java deleted file mode 100644 index 088db7df814..00000000000 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.rankeval; - -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.NamedWriteable; -import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.Writeable; - -import java.io.IOException; -import java.util.Collections; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -// TODO replace by infra from ESTestCase -public class RankEvalTestHelper { - - public static void testHashCodeAndEquals(T testItem, T mutation, T secondCopy) { - assertFalse("testItem is equal to null", testItem.equals(null)); - assertFalse("testItem is equal to incompatible type", testItem.equals("")); - assertTrue("testItem is not equal to self", testItem.equals(testItem)); - assertThat("same testItem's hashcode returns different values if called multiple times", - testItem.hashCode(), equalTo(testItem.hashCode())); - - assertThat("different testItem should not be equal", mutation, not(equalTo(testItem))); - - assertNotSame("testItem copy is not same as original", testItem, secondCopy); - assertTrue("testItem is not equal to its copy", testItem.equals(secondCopy)); - assertTrue("equals is not symmetric", secondCopy.equals(testItem)); - assertThat("testItem copy's hashcode is different from original hashcode", - secondCopy.hashCode(), equalTo(testItem.hashCode())); - } - - /** - * Make a deep copy of an object by running it through a BytesStreamOutput - * - * @param original - * the original object - * @param reader - * a function able to create a new copy of this type - * @return a new copy of the original object - */ - public static T copy(T original, Writeable.Reader reader) - throws IOException { - return copy(original, reader, new NamedWriteableRegistry(Collections.emptyList())); - } - - /** - * Make a deep copy of an object by running it through a BytesStreamOutput - * - * @param original - * the original object - * @param reader - * a function able to create a new copy of this type - * @param namedWriteableRegistry - * must be non-empty if the object itself or nested object - * implement {@link NamedWriteable} - * @return a new copy of the original object - */ - public static T copy(T original, Writeable.Reader reader, - NamedWriteableRegistry namedWriteableRegistry) throws IOException { - try (BytesStreamOutput output = new BytesStreamOutput()) { - original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), - namedWriteableRegistry)) { - return reader.read(in); - } - } - } -} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index 672d464a386..5ec9692d83a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -27,15 +28,14 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Collections; + +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; public class RatedDocumentTests extends ESTestCase { public static RatedDocument createRatedDocument() { - String index = randomAlphaOfLength(10); - String docId = randomAlphaOfLength(10); - int rating = randomInt(); - - return new RatedDocument(index, docId, rating); + return new RatedDocument(randomAlphaOfLength(10), randomAlphaOfLength(10), randomInt()); } public void testXContentParsing() throws IOException { @@ -52,22 +52,17 @@ public class RatedDocumentTests extends ESTestCase { public void testSerialization() throws IOException { RatedDocument original = createRatedDocument(); - RatedDocument deserialized = RankEvalTestHelper.copy(original, RatedDocument::new); + RatedDocument deserialized = ESTestCase.copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()), + RatedDocument::new); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); } public void testEqualsAndHash() throws IOException { - RatedDocument testItem = createRatedDocument(); - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), RankEvalTestHelper.copy(testItem, RatedDocument::new)); - } - - public void testInvalidParsing() { - expectThrows(IllegalArgumentException.class, () -> new RatedDocument(null, "abc", 10)); - expectThrows(IllegalArgumentException.class, () -> new RatedDocument("", "abc", 10)); - expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "", 10)); - expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", null, 10)); + checkEqualsAndHashCode(createRatedDocument(), original -> { + return new RatedDocument(original.getIndex(), original.getDocID(), original.getRating()); + }, RatedDocumentTests::mutateTestItem); } private static RatedDocument mutateTestItem(RatedDocument original) { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 58a9f998183..53846406d9c 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -47,6 +47,7 @@ import java.util.stream.Stream; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; public class RatedRequestsTests extends ESTestCase { @@ -140,32 +141,26 @@ public class RatedRequestsTests extends ESTestCase { for (int i = 0; i < size; i++) { indices.add(randomAlphaOfLengthBetween(0, 50)); } - RatedRequest original = createTestItem(indices, randomBoolean()); - - List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - - RatedRequest deserialized = RankEvalTestHelper.copy(original, RatedRequest::new, new NamedWriteableRegistry(namedWriteables)); + RatedRequest deserialized = copy(original); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); } + private static RatedRequest copy(RatedRequest original) throws IOException { + List namedWriteables = new ArrayList<>(); + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); + return ESTestCase.copyWriteable(original, new NamedWriteableRegistry(namedWriteables), RatedRequest::new); + } + public void testEqualsAndHash() throws IOException { List indices = new ArrayList<>(); int size = randomIntBetween(0, 20); for (int i = 0; i < size; i++) { indices.add(randomAlphaOfLengthBetween(0, 50)); } - - RatedRequest testItem = createTestItem(indices, randomBoolean()); - - List namedWriteables = new ArrayList<>(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new)); - - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - RankEvalTestHelper.copy(testItem, RatedRequest::new, new NamedWriteableRegistry(namedWriteables))); + checkEqualsAndHashCode(createTestItem(indices, randomBoolean()), RatedRequestsTests::copy, RatedRequestsTests::mutateTestItem); } private static RatedRequest mutateTestItem(RatedRequest original) { @@ -220,8 +215,7 @@ public class RatedRequestsTests extends ESTestCase { } public void testDuplicateRatedDocThrowsException() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1), - new RatedDocument(new DocumentKey("index1", "id1"), 5)); + List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1), new RatedDocument("index1", "id1", 5)); // search request set, no summary fields IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, @@ -237,45 +231,45 @@ public class RatedRequestsTests extends ESTestCase { } public void testNullSummaryFieldsTreatment() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1)); RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder()); expectThrows(IllegalArgumentException.class, () -> request.setSummaryFields(null)); } public void testNullParamsTreatment() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1)); RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, null); assertNotNull(request.getParams()); } public void testSettingParamsAndRequestThrows() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1)); Map params = new HashMap<>(); params.put("key", "value"); expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), params, null)); } public void testSettingNeitherParamsNorRequestThrows() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1)); expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null)); expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, new HashMap<>(), "templateId")); } public void testSettingParamsWithoutTemplateIdThrows() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1)); Map params = new HashMap<>(); params.put("key", "value"); expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, params, null)); } public void testSettingTemplateIdAndRequestThrows() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1)); expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, "templateId")); } public void testSettingTemplateIdNoParamsThrows() { - List ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1)); + List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1)); expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null, "templateId")); } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java index 3899a2e2029..cf66b0b7797 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.text.Text; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESTestCase; @@ -27,6 +28,8 @@ import java.io.IOException; import java.util.Collections; import java.util.Optional; +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; + public class RatedSearchHitTests extends ESTestCase { public static RatedSearchHit randomRatedSearchHit() { @@ -57,15 +60,18 @@ public class RatedSearchHitTests extends ESTestCase { public void testSerialization() throws IOException { RatedSearchHit original = randomRatedSearchHit(); - RatedSearchHit deserialized = RankEvalTestHelper.copy(original, RatedSearchHit::new); + RatedSearchHit deserialized = copy(original); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); } public void testEqualsAndHash() throws IOException { - RatedSearchHit testItem = randomRatedSearchHit(); - RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), - RankEvalTestHelper.copy(testItem, RatedSearchHit::new)); + checkEqualsAndHashCode(randomRatedSearchHit(), RatedSearchHitTests::copy, RatedSearchHitTests::mutateTestItem); } + + private static RatedSearchHit copy(RatedSearchHit original) throws IOException { + return ESTestCase.copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()), RatedSearchHit::new); + } + } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TestRatingEnum.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TestRatingEnum.java new file mode 100644 index 00000000000..ea44c215d92 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TestRatingEnum.java @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +enum TestRatingEnum { + IRRELEVANT, RELEVANT; +} \ No newline at end of file diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml index 0df0110993a..130da28f3b1 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml @@ -35,4 +35,4 @@ - match: { rank_eval.details.amsterdam_query.unknown_docs: [ ]} - match: { rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}} - - is_true: rank_eval.failures.invalid_queryy + - is_true: rank_eval.failures.invalid_query From 3348d2317fadd2d73581dfd4916f1e5ba7a514d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 15 Nov 2017 17:23:52 +0100 Subject: [PATCH 110/297] Reworking javadocs, minor changes in some implementation classes --- docs/reference/search/rank-eval.asciidoc | 6 +- .../rankeval/DiscountedCumulativeGain.java | 10 +- .../index/rankeval/EvalQueryQuality.java | 41 +++-- .../index/rankeval/EvaluationMetric.java | 4 +- .../index/rankeval/MeanReciprocalRank.java | 51 +++--- .../index/rankeval/MetricDetails.java | 3 + .../index/rankeval/PrecisionAtK.java | 58 +++---- .../index/rankeval/RankEvalAction.java | 9 +- .../index/rankeval/RankEvalPlugin.java | 13 +- .../index/rankeval/RankEvalRequest.java | 25 ++- .../rankeval/RankEvalRequestBuilder.java | 6 +- .../index/rankeval/RankEvalResponse.java | 65 +++----- .../index/rankeval/RankEvalSpec.java | 84 +++++----- .../index/rankeval/RatedDocument.java | 2 +- .../index/rankeval/RatedRequest.java | 146 ++++++++---------- .../index/rankeval/RatedSearchHit.java | 35 +---- .../index/rankeval/RestRankEvalAction.java | 18 +-- .../rankeval/TransportRankEvalAction.java | 54 +++---- .../DiscountedCumulativeGainTests.java | 1 - .../index/rankeval/EvalQueryQualityTests.java | 9 +- .../index/rankeval/RankEvalRequestIT.java | 12 +- .../index/rankeval/RankEvalResponseTests.java | 2 +- .../index/rankeval/RankEvalSpecTests.java | 33 ++-- .../index/rankeval/RatedRequestsTests.java | 82 +++------- .../rankeval}/SmokeMultipleTemplatesIT.java | 18 +-- ...stRankEvalWithMustacheYAMLTestSuiteIT.java | 2 +- 26 files changed, 340 insertions(+), 449 deletions(-) rename qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/{smoketest => index/rankeval}/SmokeMultipleTemplatesIT.java (85%) rename qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/{smoketest => index/rankeval}/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java (97%) diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index 886d262cc72..2ca1441db4a 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -62,7 +62,7 @@ The request section contains several search requests typical to your application "ratings": [ <3> { "_index": "my_index", "_id": "doc1", "rating": 0 }, { "_index": "my_index", "_id": "doc2", "rating": 3}, - { "_index": "my_index", _id": "doc3", "rating": 1 } + { "_index": "my_index", "_id": "doc3", "rating": 1 } ] }, { @@ -192,7 +192,9 @@ GET /twitter/_rank_eval "ratings": [] }], "metric": { - "mean_reciprocal_rank": {} + "mean_reciprocal_rank": { + "relevant_rating_threshold" : 1 + } } } -------------------------------- diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java index a544ffcb4ea..141d45c274b 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java @@ -38,14 +38,18 @@ import java.util.stream.Collectors; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings; +/** + * Metric implementing Discounted Cumulative Gain (https://en.wikipedia.org/wiki/Discounted_cumulative_gain).
+ * The `normalize` parameter can be set to calculate the normalized NDCG (set to false by default).
+ * The optional `unknown_doc_rating` parameter can be used to specify a default rating for unlabeled documents. + */ public class DiscountedCumulativeGain implements EvaluationMetric { /** If set to true, the dcg will be normalized (ndcg) */ private final boolean normalize; /** - * If set to, this will be the rating for docs the user hasn't supplied an - * explicit rating for + * Optional. If set, this will be the rating for docs that are unrated in the ranking evaluation request */ private final Integer unknownDocRating; @@ -69,7 +73,7 @@ public class DiscountedCumulativeGain implements EvaluationMetric { this.unknownDocRating = unknownDocRating; } - public DiscountedCumulativeGain(StreamInput in) throws IOException { + DiscountedCumulativeGain(StreamInput in) throws IOException { normalize = in.readBoolean(); unknownDocRating = in.readOptionalVInt(); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index e7e00c1699d..a42ce60c4e4 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -32,46 +32,43 @@ import java.util.List; import java.util.Objects;; /** - * This class represents the partial information from running the ranking evaluation metric on one - * request alone. It contains all information necessary to render the response for this part of the - * overall evaluation. + * Result of the evaluation metric calculation on one particular query alone. */ public class EvalQueryQuality implements ToXContent, Writeable { - /** documents seen as result for one request that were not annotated.*/ - private String id; - private double qualityLevel; + private final String queryId; + private final double evaluationResult; private MetricDetails optionalMetricDetails; - private List hits = new ArrayList<>(); + private final List hits = new ArrayList<>(); - public EvalQueryQuality(String id, double qualityLevel) { - this.id = id; - this.qualityLevel = qualityLevel; + public EvalQueryQuality(String id, double evaluationResult) { + this.queryId = id; + this.evaluationResult = evaluationResult; } public EvalQueryQuality(StreamInput in) throws IOException { this(in.readString(), in.readDouble()); - this.hits = in.readList(RatedSearchHit::new); + this.hits.addAll(in.readList(RatedSearchHit::new)); this.optionalMetricDetails = in.readOptionalNamedWriteable(MetricDetails.class); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(id); - out.writeDouble(qualityLevel); + out.writeString(queryId); + out.writeDouble(evaluationResult); out.writeList(hits); out.writeOptionalNamedWriteable(this.optionalMetricDetails); } public String getId() { - return id; + return queryId; } public double getQualityLevel() { - return qualityLevel; + return evaluationResult; } - public void addMetricDetails(MetricDetails breakdown) { + public void setMetricDetails(MetricDetails breakdown) { this.optionalMetricDetails = breakdown; } @@ -80,7 +77,7 @@ public class EvalQueryQuality implements ToXContent, Writeable { } public void addHitsAndRatings(List hits) { - this.hits = hits; + this.hits.addAll(hits); } public List getHitsAndRatings() { @@ -89,8 +86,8 @@ public class EvalQueryQuality implements ToXContent, Writeable { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(id); - builder.field("quality_level", this.qualityLevel); + builder.startObject(queryId); + builder.field("quality_level", this.evaluationResult); builder.startArray("unknown_docs"); for (DocumentKey key : EvaluationMetric.filterUnknownDocuments(hits)) { builder.startObject(); @@ -122,14 +119,14 @@ public class EvalQueryQuality implements ToXContent, Writeable { return false; } EvalQueryQuality other = (EvalQueryQuality) obj; - return Objects.equals(id, other.id) && - Objects.equals(qualityLevel, other.qualityLevel) && + return Objects.equals(queryId, other.queryId) && + Objects.equals(evaluationResult, other.evaluationResult) && Objects.equals(hits, other.hits) && Objects.equals(optionalMetricDetails, other.optionalMetricDetails); } @Override public final int hashCode() { - return Objects.hash(id, qualityLevel, hits, optionalMetricDetails); + return Objects.hash(queryId, evaluationResult, hits, optionalMetricDetails); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java index 6754d039266..865093e9a9e 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java @@ -87,8 +87,10 @@ public interface EvaluationMetric extends ToXContent, NamedWriteable { return rc; } + /** + * join hits with rated documents using the joint _index/_id document key + */ static List joinHitsWithRatings(SearchHit[] hits, List ratedDocs) { - // join hits with rated documents Map ratedDocumentMap = ratedDocs.stream() .collect(Collectors.toMap(RatedDocument::getKey, item -> item)); List ratedSearchHits = new ArrayList<>(hits.length); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java index 2b47f68587e..057bff6e147 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java @@ -32,16 +32,13 @@ import java.util.List; import java.util.Objects; import java.util.Optional; -import javax.naming.directory.SearchResult; - import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings; /** - * Evaluates using mean reciprocal rank. By default documents with a rating - * equal or bigger than 1 are considered to be "relevant" for the reciprocal - * rank calculation. This value can be changes using the - * "relevant_rating_threshold" parameter. + * Metric implementing Mean Reciprocal Rank (https://en.wikipedia.org/wiki/Mean_reciprocal_rank).
+ * By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the reciprocal + * rank calculation. This value can be changes using the relevant_rating_threshold` parameter. */ public class MeanReciprocalRank implements EvaluationMetric { @@ -56,16 +53,24 @@ public class MeanReciprocalRank implements EvaluationMetric { this(DEFAULT_RATING_THRESHOLD); } - public MeanReciprocalRank(StreamInput in) throws IOException { + MeanReciprocalRank(StreamInput in) throws IOException { this.relevantRatingThreshhold = in.readVInt(); } - public MeanReciprocalRank(int threshold) { - if (threshold < 0) { - throw new IllegalArgumentException( - "Relevant rating threshold for precision must be positive integer."); + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(relevantRatingThreshhold); + } + + /** + * Metric implementing Mean Reciprocal Rank (https://en.wikipedia.org/wiki/Mean_reciprocal_rank).
+ * @param relevantRatingThreshold the rating value that a document needs to be regarded as "relevalnt". Defaults to 1. + */ + public MeanReciprocalRank(int relevantRatingThreshold) { + if (relevantRatingThreshold < 0) { + throw new IllegalArgumentException("Relevant rating threshold for precision must be positive integer."); } - this.relevantRatingThreshhold = threshold; + this.relevantRatingThreshhold = relevantRatingThreshold; } @Override @@ -74,8 +79,7 @@ public class MeanReciprocalRank implements EvaluationMetric { } /** - * Return the rating threshold above which ratings are considered to be - * "relevant" for this metric. Defaults to 1. + * Return the rating threshold above which ratings are considered to be "relevant". */ public int getRelevantRatingThreshold() { return relevantRatingThreshhold; @@ -83,8 +87,6 @@ public class MeanReciprocalRank implements EvaluationMetric { /** * Compute ReciprocalRank based on provided relevant document IDs. - * - * @return reciprocal Rank for above {@link SearchResult} list. **/ @Override public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, @@ -105,16 +107,11 @@ public class MeanReciprocalRank implements EvaluationMetric { double reciprocalRank = (firstRelevant == -1) ? 0 : 1.0d / firstRelevant; EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, reciprocalRank); - evalQueryQuality.addMetricDetails(new Breakdown(firstRelevant)); + evalQueryQuality.setMetricDetails(new Breakdown(firstRelevant)); evalQueryQuality.addHitsAndRatings(ratedHits); return evalQueryQuality; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(relevantRatingThreshhold); - } - private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("reciprocal_rank", args -> { @@ -161,15 +158,15 @@ public class MeanReciprocalRank implements EvaluationMetric { return Objects.hash(relevantRatingThreshhold); } - public static class Breakdown implements MetricDetails { + static class Breakdown implements MetricDetails { - private int firstRelevantRank; + private final int firstRelevantRank; - public Breakdown(int firstRelevantRank) { + Breakdown(int firstRelevantRank) { this.firstRelevantRank = firstRelevantRank; } - public Breakdown(StreamInput in) throws IOException { + Breakdown(StreamInput in) throws IOException { this.firstRelevantRank = in.readVInt(); } @@ -190,7 +187,7 @@ public class MeanReciprocalRank implements EvaluationMetric { return NAME; } - public int getFirstRelevantRank() { + int getFirstRelevantRank() { return firstRelevantRank; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java index af838111427..22b0ed19cf2 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java @@ -22,6 +22,9 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.xcontent.ToXContent; +/** + * Details about a specific {@link EvaluationMetric} that should be included in the resonse. + */ public interface MetricDetails extends ToXContent, NamedWriteable { } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java index 76cc72bc82d..9152141043c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java @@ -38,10 +38,13 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optiona import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings; /** - * Evaluate Precision of the search results. Documents without a rating are - * ignored. By default documents with a rating equal or bigger than 1 are - * considered to be "relevant" for the precision calculation. This value can be - * changes using the "relevant_rating_threshold" parameter. + * Metric implementing Precision@K + * (https://en.wikipedia.org/wiki/Information_retrieval#Precision_at_K).
+ * By default documents with a rating equal or bigger than 1 are considered to + * be "relevant" for this calculation. This value can be changes using the + * relevant_rating_threshold` parameter.
+ * The `ignore_unlabeled` parameter (default to false) controls if unrated + * documents should be ignored. */ public class PrecisionAtK implements EvaluationMetric { @@ -50,24 +53,25 @@ public class PrecisionAtK implements EvaluationMetric { private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); private static final ParseField IGNORE_UNLABELED_FIELD = new ParseField("ignore_unlabeled"); - /** - * This setting controls how unlabeled documents in the search hits are - * treated. Set to 'true', unlabeled documents are ignored and neither count - * as true or false positives. Set to 'false', they are treated as false - * positives. - */ private final boolean ignoreUnlabeled; - - /** ratings equal or above this value will be considered relevant. */ private final int relevantRatingThreshhold; + /** + * Metric implementing Precision@K. + * @param threshold + * ratings equal or above this value will be considered relevant. + * @param ignoreUnlabeled + * Controls how unlabeled documents in the search hits are treated. + * Set to 'true', unlabeled documents are ignored and neither count + * as true or false positives. Set to 'false', they are treated as + * false positives. + */ public PrecisionAtK(int threshold, boolean ignoreUnlabeled) { - if (threshold < 0) { - throw new IllegalArgumentException( - "Relevant rating threshold for precision must be positive integer."); - } - this.relevantRatingThreshhold = threshold; - this.ignoreUnlabeled = ignoreUnlabeled; + if (threshold < 0) { + throw new IllegalArgumentException("Relevant rating threshold for precision must be positive integer."); + } + this.relevantRatingThreshhold = threshold; + this.ignoreUnlabeled = ignoreUnlabeled; } public PrecisionAtK() { @@ -113,7 +117,7 @@ public class PrecisionAtK implements EvaluationMetric { } /** - * Gets the 'ìgnore_unlabeled' parameter + * Gets the 'ignore_unlabeled' parameter. */ public boolean getIgnoreUnlabeled() { return ignoreUnlabeled; @@ -151,7 +155,7 @@ public class PrecisionAtK implements EvaluationMetric { precision = (double) truePositives / (truePositives + falsePositives); } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision); - evalQueryQuality.addMetricDetails( + evalQueryQuality.setMetricDetails( new PrecisionAtK.Breakdown(truePositives, truePositives + falsePositives)); evalQueryQuality.addHitsAndRatings(ratedSearchHits); return evalQueryQuality; @@ -186,19 +190,19 @@ public class PrecisionAtK implements EvaluationMetric { return Objects.hash(relevantRatingThreshhold, ignoreUnlabeled); } - public static class Breakdown implements MetricDetails { + static class Breakdown implements MetricDetails { - public static final String DOCS_RETRIEVED_FIELD = "docs_retrieved"; - public static final String RELEVANT_DOCS_RETRIEVED_FIELD = "relevant_docs_retrieved"; + private static final String DOCS_RETRIEVED_FIELD = "docs_retrieved"; + private static final String RELEVANT_DOCS_RETRIEVED_FIELD = "relevant_docs_retrieved"; private int relevantRetrieved; private int retrieved; - public Breakdown(int relevantRetrieved, int retrieved) { + Breakdown(int relevantRetrieved, int retrieved) { this.relevantRetrieved = relevantRetrieved; this.retrieved = retrieved; } - public Breakdown(StreamInput in) throws IOException { + Breakdown(StreamInput in) throws IOException { this.relevantRetrieved = in.readVInt(); this.retrieved = in.readVInt(); } @@ -222,11 +226,11 @@ public class PrecisionAtK implements EvaluationMetric { return NAME; } - public int getRelevantRetrieved() { + int getRelevantRetrieved() { return relevantRetrieved; } - public int getRetrieved() { + int getRetrieved() { return retrieved; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java index b2e4aeb8777..8908fbdfbdd 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java @@ -23,13 +23,12 @@ import org.elasticsearch.action.Action; import org.elasticsearch.client.ElasticsearchClient; /** - * Action used to start precision at qa evaluations. - **/ -public class RankEvalAction - extends Action { + * Action for explaining evaluating search ranking results. + */ +public class RankEvalAction extends Action { public static final RankEvalAction INSTANCE = new RankEvalAction(); - public static final String NAME = "indices:data/read/quality"; + public static final String NAME = "indices:data/read/rank_eval"; private RankEvalAction() { super(NAME); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index 189f7ab91ac..53c976392ce 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -43,22 +42,16 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin { @Override public List> getActions() { - return Arrays.asList( - new ActionHandler<>(RankEvalAction.INSTANCE, TransportRankEvalAction.class)); + return Arrays.asList(new ActionHandler<>(RankEvalAction.INSTANCE, TransportRankEvalAction.class)); } @Override - public List getRestHandlers(Settings settings, RestController restController, - ClusterSettings clusterSettings, IndexScopedSettings indexScopedSettings, - SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, + public List getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster) { return Arrays.asList(new RestRankEvalAction(settings, restController)); } - /** - * Returns parsers for {@link NamedWriteable} objects that this plugin sends over the transport protocol. - * @see NamedWriteableRegistry - */ @Override public List getNamedWriteables() { List namedWriteables = new ArrayList<>(); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java index 2d75bc88671..637b9a18844 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java @@ -27,51 +27,46 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; /** - * Instances of this class represent a complete precision at request. They - * encode a precision task including search intents and search specifications to - * be executed subsequently. + * Request to perform a search ranking evaluation. */ public class RankEvalRequest extends ActionRequest { - /** The request data to use for evaluation. */ - private RankEvalSpec task; + private RankEvalSpec rankingEvaluation; @Override public ActionRequestValidationException validate() { ActionRequestValidationException e = null; - if (task == null) { + if (rankingEvaluation == null) { e = new ActionRequestValidationException(); e.addValidationError("missing ranking evaluation specification"); } - return null; + return e; } /** - * Returns the specification of this qa run including intents to execute, - * specifications detailing intent translation and metrics to compute. + * Returns the specification of the ranking evaluation. */ public RankEvalSpec getRankEvalSpec() { - return task; + return rankingEvaluation; } /** - * Returns the specification of this qa run including intents to execute, - * specifications detailing intent translation and metrics to compute. + * Set the the specification of the ranking evaluation. */ public void setRankEvalSpec(RankEvalSpec task) { - this.task = task; + this.rankingEvaluation = task; } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - task = new RankEvalSpec(in); + rankingEvaluation = new RankEvalSpec(in); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - task.writeTo(out); + rankingEvaluation.writeTo(out); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java index 77306259fe4..2df16ade566 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java @@ -23,11 +23,9 @@ import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -public class RankEvalRequestBuilder - extends ActionRequestBuilder { +public class RankEvalRequestBuilder extends ActionRequestBuilder { - public RankEvalRequestBuilder(ElasticsearchClient client, - Action action, + public RankEvalRequestBuilder(ElasticsearchClient client, Action action, RankEvalRequest request) { super(client, action, request); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index b5904422f04..30ffaeff18b 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -32,48 +33,32 @@ import java.util.HashMap; import java.util.Map; /** - * For each qa specification identified by its id this response returns the - * respective averaged precisionAnN value. - * - * In addition for each query the document ids that haven't been found annotated - * is returned as well. - * - * Documents of unknown quality - i.e. those that haven't been supplied in the - * set of annotated documents but have been returned by the search are not taken - * into consideration when computing precision at n - they are ignored. - * + * Returns the results for a {@link RankEvalRequest}.
+ * The repsonse contains a detailed section for each evaluation query in the request and + * possible failures that happened when executin individual queries. **/ -// TODO instead of just returning averages over complete results, think of other -// statistics, micro avg, macro avg, partial results public class RankEvalResponse extends ActionResponse implements ToXContentObject { - /** - * Average precision observed when issuing query intents with this - * specification. - */ - private double qualityLevel; - /** - * Mapping from intent id to all documents seen for this intent that were - * not annotated. - */ - private Map details; - /** - * Mapping from intent id to potential exceptions that were thrown on query - * execution. - */ - private Map failures; - public RankEvalResponse() { - } + /** The overall evaluation result. */ + private double evaluationResult; + /** details about individual ranking evaluation queries, keyed by their id */ + private Map details; + /** exceptions for specific ranking evaluation queries, keyed by their id */ + private Map failures; public RankEvalResponse(double qualityLevel, Map partialResults, Map failures) { - this.qualityLevel = qualityLevel; - this.details = partialResults; - this.failures = failures; + this.evaluationResult = qualityLevel; + this.details = new HashMap<>(partialResults); + this.failures = new HashMap<>(failures); } - public double getQualityLevel() { - return qualityLevel; + RankEvalResponse() { + // only used in RankEvalAction#newResponse() + } + + public double getEvaluationResult() { + return evaluationResult; } public Map getPartialResults() { @@ -86,14 +71,13 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject @Override public String toString() { - return "RankEvalResponse, quality: " + qualityLevel + ", partial results: " + details - + ", number of failures: " + failures.size(); + return Strings.toString(this); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeDouble(qualityLevel); + out.writeDouble(evaluationResult); out.writeVInt(details.size()); for (String queryId : details.keySet()) { out.writeString(queryId); @@ -109,7 +93,7 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - this.qualityLevel = in.readDouble(); + this.evaluationResult = in.readDouble(); int partialResultSize = in.readVInt(); this.details = new HashMap<>(partialResultSize); for (int i = 0; i < partialResultSize; i++) { @@ -129,7 +113,7 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.startObject("rank_eval"); - builder.field("quality_level", qualityLevel); + builder.field("quality_level", evaluationResult); builder.startObject("details"); for (String key : details.keySet()) { details.get(key).toXContent(builder, params); @@ -138,8 +122,7 @@ public class RankEvalResponse extends ActionResponse implements ToXContentObject builder.startObject("failures"); for (String key : failures.keySet()) { builder.startObject(key); - ElasticsearchException.generateFailureXContent(builder, params, failures.get(key), - true); + ElasticsearchException.generateFailureXContent(builder, params, failures.get(key), false); builder.endObject(); } builder.endObject(); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 3a88bc4de57..cae326ad1a2 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -42,19 +42,12 @@ import java.util.Map.Entry; import java.util.Objects; /** - * This class defines a ranking evaluation task including an id, a collection of - * queries to evaluate and the evaluation metric. - * - * Each QA run is based on a set of queries to send to the index and multiple QA - * specifications that define how to translate the query intents into elastic - * search queries. + * Specification of the ranking evaluation request.
+ * This class groups the queries to evaluate, including their document ratings, + * and the evaluation metric including its parameters. */ - public class RankEvalSpec implements Writeable, ToXContentObject { - /** - * Collection of query specifications, that is e.g. search request templates - * to use for query translation. - */ + /** List of search request to use for the evaluation */ private final List ratedRequests; /** Definition of the quality metric, e.g. precision at N */ private final EvaluationMetric metric; @@ -64,34 +57,30 @@ public class RankEvalSpec implements Writeable, ToXContentObject { private static final int MAX_CONCURRENT_SEARCHES = 10; /** optional: Templates to base test requests on */ private Map templates = new HashMap<>(); + /** the indices this ranking evaluation targets */ + private final List indices; - public RankEvalSpec(List ratedRequests, EvaluationMetric metric, - Collection templates) { - if (ratedRequests == null || ratedRequests.size() < 1) { - throw new IllegalStateException( - "Cannot evaluate ranking if no search requests with rated results are provided." - + " Seen: " + ratedRequests); + public RankEvalSpec(List ratedRequests, EvaluationMetric metric, Collection templates) { + this.metric = Objects.requireNonNull(metric, "Cannot evaluate ranking if no evaluation metric is provided."); + if (ratedRequests == null || ratedRequests.isEmpty()) { + throw new IllegalArgumentException( + "Cannot evaluate ranking if no search requests with rated results are provided. Seen: " + ratedRequests); } - if (metric == null) { - throw new IllegalStateException( - "Cannot evaluate ranking if no evaluation metric is provided."); - } - if (templates == null || templates.size() < 1) { + this.ratedRequests = ratedRequests; + if (templates == null || templates.isEmpty()) { for (RatedRequest request : ratedRequests) { if (request.getTestRequest() == null) { - throw new IllegalStateException( - "Cannot evaluate ranking if neither template nor test request is " - + "provided. Seen for request id: " + request.getId()); + throw new IllegalStateException("Cannot evaluate ranking if neither template nor test request is " + + "provided. Seen for request id: " + request.getId()); } } } - this.ratedRequests = ratedRequests; - this.metric = metric; if (templates != null) { for (ScriptWithId idScript : templates) { this.templates.put(idScript.id, idScript.script); } } + this.indices = new ArrayList<>(); } public RankEvalSpec(List ratedRequests, EvaluationMetric metric) { @@ -112,6 +101,11 @@ public class RankEvalSpec implements Writeable, ToXContentObject { this.templates.put(key, value); } maxConcurrentSearches = in.readVInt(); + int indicesSize = in.readInt(); + indices = new ArrayList<>(indicesSize); + for (int i = 0; i < indicesSize; i++) { + this.indices.add(in.readString()); + } } @Override @@ -127,6 +121,10 @@ public class RankEvalSpec implements Writeable, ToXContentObject { entry.getValue().writeTo(out); } out.writeVInt(maxConcurrentSearches); + out.writeInt(indices.size()); + for (String index : indices) { + out.writeString(index); + } } /** Returns the metric to use for quality evaluation.*/ @@ -154,30 +152,35 @@ public class RankEvalSpec implements Writeable, ToXContentObject { this.maxConcurrentSearches = maxConcurrentSearches; } + public void addIndices(List indices) { + this.indices.addAll(indices); + } + + public List getIndices() { + return Collections.unmodifiableList(indices); + } + private static final ParseField TEMPLATES_FIELD = new ParseField("templates"); private static final ParseField METRIC_FIELD = new ParseField("metric"); private static final ParseField REQUESTS_FIELD = new ParseField("requests"); - private static final ParseField MAX_CONCURRENT_SEARCHES_FIELD - = new ParseField("max_concurrent_searches"); + private static final ParseField MAX_CONCURRENT_SEARCHES_FIELD = new ParseField("max_concurrent_searches"); @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("rank_eval", - a -> new RankEvalSpec((List) a[0], - (EvaluationMetric) a[1], (Collection) a[2])); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rank_eval", + a -> new RankEvalSpec((List) a[0], (EvaluationMetric) a[1], (Collection) a[2])); static { PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> { - return RatedRequest.fromXContent(p); - } , REQUESTS_FIELD); + return RatedRequest.fromXContent(p); + }, REQUESTS_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { try { return EvaluationMetric.fromXContent(p); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } - } , METRIC_FIELD); + }, METRIC_FIELD); PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { - return ScriptWithId.fromXContent(p); + return ScriptWithId.fromXContent(p); }, TEMPLATES_FIELD); PARSER.declareInt(RankEvalSpec::setMaxConcurrentSearches, MAX_CONCURRENT_SEARCHES_FIELD); } @@ -186,14 +189,14 @@ public class RankEvalSpec implements Writeable, ToXContentObject { return PARSER.apply(parser, null); } - public static class ScriptWithId { + static class ScriptWithId { private Script script; private String id; private static final ParseField TEMPLATE_FIELD = new ParseField("template"); private static final ParseField TEMPLATE_ID_FIELD = new ParseField("id"); - public ScriptWithId(String id, Script script) { + ScriptWithId(String id, Script script) { this.id = id; this.script = script; } @@ -212,8 +215,7 @@ public class RankEvalSpec implements Writeable, ToXContentObject { try { return Script.parse(p, "mustache"); } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing rank request", - ex); + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } }, TEMPLATE_FIELD); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index fc5880e60cd..6fe6a96a9d3 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -36,7 +36,7 @@ import java.util.Objects; * Represents a document (specified by its _index/_id) and its corresponding rating * with respect to a specific search query. *

- * Json structure in a request: + * The json structure of this element in a request: * 

  * {
  *   "_index": "my_index",
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java
index e8ed925987f..3426a6b9be4 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java
@@ -34,6 +34,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -43,36 +44,49 @@ import java.util.Objects;
 import java.util.Set;
 
 /**
- * Defines a QA specification: All end user supplied query intents will be
- * mapped to the search request specified in this search request template and
- * executed against the targetIndex given. Any filters that should be applied in
- * the target system can be specified as well.
- *
- * The resulting document lists can then be compared against what was specified
- * in the set of rated documents as part of a QAQuery.
+ * Definition of a particular query in the ranking evaluation request.

+ * This usually represents a single user search intent and consists of an id
+ * (ideally human readable and referencing the search intent), the list of
+ * indices to be queries and the {@link SearchSourceBuilder} that will be used
+ * to create the search request for this search intent.

+ * Alternatively, a template id and template parameters can be provided instead.

+ * Finally, a list of rated documents for this query also needs to be provided.
+ * 

+ * The json structure in the rest request looks like this:
+ * 
+ * {
+ *    "id": "coffee_query",
+ *    "request": {
+ *        "query": {
+ *            "match": { "beverage": "coffee" }
+ *        }
+ *    },
+ *    "summary_fields": ["title"],
+ *    "ratings": [
+ *        {"_index": "my_index", "_id": "doc1", "rating": 0},
+ *        {"_index": "my_index", "_id": "doc2","rating": 3},
+ *        {"_index": "my_index", "_id": "doc3", "rating": 1}
+ *    ]
+ * }
+ * 

  */
 @SuppressWarnings("unchecked")
 public class RatedRequest implements Writeable, ToXContentObject {
-    private String id;
-    private List indices = new ArrayList<>();
-    private List summaryFields = new ArrayList<>();
-    /** Collection of rated queries for this query QA specification. */
-    private List ratedDocs = new ArrayList<>();
-    /**
-     * Search request to execute for this rated request, can be null, if
-     * template and corresponding params are supplied.
-     */
+    private final String id;
+    private final List summaryFields;
+    private final List ratedDocs;
+    // Search request to execute for this rated request. This can be null if template and corresponding parameters are supplied.
     @Nullable
     private SearchSourceBuilder testRequest;
     /**
      * Map of parameters to use for filling a query template, can be used
      * instead of providing testRequest.
      */
-    private Map params = new HashMap<>();
+    private final Map params;
     @Nullable
     private String templateId;
 
-    public RatedRequest(String id, List ratedDocs, SearchSourceBuilder testRequest,
+    private RatedRequest(String id, List ratedDocs, SearchSourceBuilder testRequest,
             Map params, String templateId) {
         if (params != null && (params.size() > 0 && testRequest != null)) {
             throw new IllegalArgumentException(
@@ -93,24 +107,26 @@ public class RatedRequest implements Writeable, ToXContentObject {
                     "If template parameters are supplied need to set id of template to apply "
                     + "them to too.");
         }
-        // No documents with same _index/id allowed.
+        // check that not two documents with same _index/id are specified
         Set docKeys = new HashSet<>();
         for (RatedDocument doc : ratedDocs) {
             if (docKeys.add(doc.getKey()) == false) {
-                String docKeyToString = doc.getKey().toString().replaceAll("\n", "")
-                        .replaceAll("  ", " ");
+                String docKeyToString = doc.getKey().toString().replaceAll("\n", "").replaceAll("  ", " ");
                 throw new IllegalArgumentException(
-                        "Found duplicate rated document key [" + docKeyToString + "]");
+                        "Found duplicate rated document key [" + docKeyToString + "] in evaluation request [" + id + "]");
             }
         }
 
         this.id = id;
         this.testRequest = testRequest;
-        this.ratedDocs = ratedDocs;
+        this.ratedDocs = new ArrayList<>(ratedDocs);
         if (params != null) {
-            this.params = params;
+            this.params = new HashMap<>(params);
+        } else {
+            this.params = Collections.emptyMap();
         }
         this.templateId = templateId;
+        this.summaryFields = new ArrayList<>();
     }
 
     public RatedRequest(String id, List ratedDocs, Map params,
@@ -126,11 +142,6 @@ public class RatedRequest implements Writeable, ToXContentObject {
         this.id = in.readString();
         testRequest = in.readOptionalWriteable(SearchSourceBuilder::new);
 
-        int indicesSize = in.readInt();
-        indices = new ArrayList<>(indicesSize);
-        for (int i = 0; i < indicesSize; i++) {
-            this.indices.add(in.readString());
-        }
         int intentSize = in.readInt();
         ratedDocs = new ArrayList<>(intentSize);
         for (int i = 0; i < intentSize; i++) {
@@ -150,10 +161,6 @@ public class RatedRequest implements Writeable, ToXContentObject {
         out.writeString(id);
         out.writeOptionalWriteable(testRequest);
 
-        out.writeInt(indices.size());
-        for (String index : indices) {
-            out.writeString(index);
-        }
         out.writeInt(ratedDocs.size());
         for (RatedDocument ratedDoc : ratedDocs) {
             ratedDoc.writeTo(out);
@@ -170,45 +177,33 @@ public class RatedRequest implements Writeable, ToXContentObject {
         return testRequest;
     }
 
-    public void setIndices(List indices) {
-        this.indices = indices;
-    }
-
-    public List getIndices() {
-        return indices;
-    }
-
-    /** Returns a user supplied spec id for easier referencing. */
+    /** return the user supplied request id */
     public String getId() {
         return id;
     }
 
-    /** Returns a list of rated documents to evaluate. */
+    /** return the list of rated documents to evaluate. */
     public List getRatedDocs() {
-        return ratedDocs;
+        return Collections.unmodifiableList(ratedDocs);
     }
 
+    /** return the parameters if this request uses a template, otherwise this will be empty. */
     public Map getParams() {
-        return this.params;
+        return Collections.unmodifiableMap(this.params);
     }
 
+    /** return the parameters if this request uses a template, otherwise this will be null. */
     public String getTemplateId() {
         return this.templateId;
     }
 
-    /**
-     * Returns a list of fields that are included in the docs summary of matched
-     * documents.
-     */
+    /** returns a list of fields that should be included in the document summary for matched documents */
     public List getSummaryFields() {
-        return summaryFields;
+        return Collections.unmodifiableList(summaryFields);
     }
 
-    public void setSummaryFields(List summaryFields) {
-        if (summaryFields == null) {
-            throw new IllegalArgumentException("Setting summaryFields to null not allowed.");
-        }
-        this.summaryFields = summaryFields;
+    public void addSummaryFields(List summaryFields) {
+        this.summaryFields.addAll(Objects.requireNonNull(summaryFields, "no summary fields supplied"));
     }
 
     private static final ParseField ID_FIELD = new ParseField("id");
@@ -218,10 +213,9 @@ public class RatedRequest implements Writeable, ToXContentObject {
     private static final ParseField FIELDS_FIELD = new ParseField("summary_fields");
     private static final ParseField TEMPLATE_ID_FIELD = new ParseField("template_id");
 
-    private static final ConstructingObjectParser PARSER =
-            new ConstructingObjectParser<>("requests",
-                    a -> new RatedRequest((String) a[0], (List) a[1],
-                            (SearchSourceBuilder) a[2], (Map) a[3], (String) a[4]));
+    private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("requests",
+            a -> new RatedRequest((String) a[0], (List) a[1], (SearchSourceBuilder) a[2], (Map) a[3],
+                    (String) a[4]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD);
@@ -230,20 +224,13 @@ public class RatedRequest implements Writeable, ToXContentObject {
         }, RATINGS_FIELD);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) ->
                 SearchSourceBuilder.fromXContent(p), REQUEST_FIELD);
-        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> (Map) p.map(), PARAMS_FIELD);
-        PARSER.declareStringArray(RatedRequest::setSummaryFields, FIELDS_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), PARAMS_FIELD);
+        PARSER.declareStringArray(RatedRequest::addSummaryFields, FIELDS_FIELD);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TEMPLATE_ID_FIELD);
     }
 
     /**
-     * Parses {@link RatedRequest} from rest representation:
-     *
-     * Example: { "id": "coffee_query", "request": { "query": { "bool": {
-     * "must": [ {"match": {"beverage": "coffee"}}, {"term": {"browser":
-     * {"value": "safari"}}}, {"term": {"time_of_day": {"value":
-     * "morning","boost": 2}}}, {"term": {"ip_location": {"value":
-     * "ams","boost": 10}}}]} }, "size": 10 }, "summary_fields" : ["body"],
-     * "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ] }
+     * parse from rest representation
      */
     public static RatedRequest fromXContent(XContentParser parser) {
         return PARSER.apply(parser, null);
@@ -256,16 +243,21 @@ public class RatedRequest implements Writeable, ToXContentObject {
         if (testRequest != null) {
             builder.field(REQUEST_FIELD.getPreferredName(), this.testRequest);
         }
-        builder.startObject(PARAMS_FIELD.getPreferredName());
-        for (Entry entry : this.params.entrySet()) {
-            builder.field(entry.getKey(), entry.getValue());
-        }
-        builder.endObject();
         builder.startArray(RATINGS_FIELD.getPreferredName());
         for (RatedDocument doc : this.ratedDocs) {
             doc.toXContent(builder, params);
         }
         builder.endArray();
+        if (this.templateId != null) {
+            builder.field(TEMPLATE_ID_FIELD.getPreferredName(), this.templateId);
+        }
+        if (this.params.isEmpty() == false) {
+            builder.startObject(PARAMS_FIELD.getPreferredName());
+            for (Entry entry : this.params.entrySet()) {
+                builder.field(entry.getKey(), entry.getValue());
+            }
+            builder.endObject();
+        }
         if (this.summaryFields.isEmpty() == false) {
             builder.startArray(FIELDS_FIELD.getPreferredName());
             for (String field : this.summaryFields) {
@@ -273,9 +265,6 @@ public class RatedRequest implements Writeable, ToXContentObject {
             }
             builder.endArray();
         }
-        if (this.templateId != null) {
-            builder.field(TEMPLATE_ID_FIELD.getPreferredName(), this.templateId);
-        }
         builder.endObject();
         return builder;
     }
@@ -297,7 +286,6 @@ public class RatedRequest implements Writeable, ToXContentObject {
         RatedRequest other = (RatedRequest) obj;
 
         return Objects.equals(id, other.id) && Objects.equals(testRequest, other.testRequest)
-                && Objects.equals(indices, other.indices)
                 && Objects.equals(summaryFields, other.summaryFields)
                 && Objects.equals(ratedDocs, other.ratedDocs)
                 && Objects.equals(params, other.params)
@@ -306,7 +294,7 @@ public class RatedRequest implements Writeable, ToXContentObject {
 
     @Override
     public final int hashCode() {
-        return Objects.hash(id, testRequest, indices, summaryFields, ratedDocs, params,
+        return Objects.hash(id, testRequest, summaryFields, ratedDocs, params,
                 templateId);
     }
 }
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java
index 9f5838e070f..11c76f7fb30 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java
@@ -24,13 +24,15 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.search.SearchHit;
 
 import java.io.IOException;
 import java.util.Objects;
 import java.util.Optional;
 
+/**
+ * Combines a {@link SearchHit} with a document rating.
+ */
 public class RatedSearchHit implements Writeable, ToXContent {
 
     private final SearchHit searchHit;
@@ -41,7 +43,7 @@ public class RatedSearchHit implements Writeable, ToXContent {
         this.rating = rating;
     }
 
-    public RatedSearchHit(StreamInput in) throws IOException {
+    RatedSearchHit(StreamInput in) throws IOException {
         this(SearchHit.readSearchHit(in),
                 in.readBoolean() == true ? Optional.of(in.readVInt()) : Optional.empty());
     }
@@ -82,37 +84,12 @@ public class RatedSearchHit implements Writeable, ToXContent {
             return false;
         }
         RatedSearchHit other = (RatedSearchHit) obj;
-        // NORELEASE this is a workaround because InternalSearchHit does not
-        // properly implement equals()/hashCode(), so we compare their
-        // xcontent
-        XContentBuilder builder;
-        String hitAsXContent;
-        String otherHitAsXContent;
-        try {
-            builder = XContentFactory.jsonBuilder();
-            hitAsXContent = searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS).string();
-            builder = XContentFactory.jsonBuilder();
-            otherHitAsXContent = other.searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS)
-                    .string();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
         return Objects.equals(rating, other.rating)
-                && Objects.equals(hitAsXContent, otherHitAsXContent);
+                && Objects.equals(searchHit, other.searchHit);
     }
 
     @Override
     public final int hashCode() {
-        // NORELEASE for this to work requires InternalSearchHit to properly
-        // implement equals()/hashCode()
-        XContentBuilder builder;
-        String hitAsXContent;
-        try {
-            builder = XContentFactory.jsonBuilder();
-            hitAsXContent = searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS).string();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return Objects.hash(rating, hitAsXContent);
+        return Objects.hash(rating, searchHit);
     }
 }
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java
index c652f7d5e3a..1efe5b4e39b 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java
@@ -98,8 +98,7 @@ public class RestRankEvalAction extends BaseRestHandler {
     }
 
     @Override
-    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client)
-            throws IOException {
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         RankEvalRequest rankEvalRequest = new RankEvalRequest();
         try (XContentParser parser = request.contentOrSourceParamParser()) {
             parseRankEvalRequest(rankEvalRequest, request, parser);
@@ -108,17 +107,10 @@ public class RestRankEvalAction extends BaseRestHandler {
                 new RestToXContentListener(channel));
     }
 
-    private static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request,
-            XContentParser parser) {
-        List indices = Arrays
-                .asList(Strings.splitStringByCommaToArray(request.param("index")));
-        RankEvalSpec spec = null;
-        spec = RankEvalSpec.parse(parser);
-        for (RatedRequest specification : spec.getRatedRequests()) {
-            specification.setIndices(indices);
-        }
-        ;
-
+    private static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, XContentParser parser) {
+        List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index")));
+        RankEvalSpec spec = RankEvalSpec.parse(parser);
+        spec.addIndices(indices);
         rankEvalRequest.setRankEvalSpec(spec);
     }
 
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
index 4e3fa8afa46..658cdbd0d68 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
@@ -66,20 +66,18 @@ import static org.elasticsearch.common.xcontent.XContentHelper.createParser;
  * returned for each search specification for the full set of search intents as
  * averaged precision at n.
  */
-public class TransportRankEvalAction
-        extends HandledTransportAction {
+public class TransportRankEvalAction extends HandledTransportAction {
     private Client client;
     private ScriptService scriptService;
     Queue taskQueue = new ConcurrentLinkedQueue<>();
     private NamedXContentRegistry namedXContentRegistry;
 
     @Inject
-    public TransportRankEvalAction(Settings settings, ThreadPool threadPool,
-            ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-            Client client, TransportService transportService, ScriptService scriptService,
-            NamedXContentRegistry namedXContentRegistry) {
-        super(settings, RankEvalAction.NAME, threadPool, transportService, actionFilters,
-                indexNameExpressionResolver, RankEvalRequest::new);
+    public TransportRankEvalAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
+            IndexNameExpressionResolver indexNameExpressionResolver, Client client, TransportService transportService,
+            ScriptService scriptService, NamedXContentRegistry namedXContentRegistry) {
+        super(settings, RankEvalAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
+                RankEvalRequest::new);
         this.scriptService = scriptService;
         this.namedXContentRegistry = namedXContentRegistry;
         this.client = client;
@@ -87,23 +85,21 @@ public class TransportRankEvalAction
 
     @Override
     protected void doExecute(RankEvalRequest request, ActionListener listener) {
-        RankEvalSpec qualityTask = request.getRankEvalSpec();
+        RankEvalSpec evaluationSpecification = request.getRankEvalSpec();
+        List indices = evaluationSpecification.getIndices();
 
-        Collection ratedRequests = qualityTask.getRatedRequests();
+        Collection ratedRequests = evaluationSpecification.getRatedRequests();
         AtomicInteger responseCounter = new AtomicInteger(ratedRequests.size());
         Map partialResults = new ConcurrentHashMap<>(
                 ratedRequests.size());
         Map errors = new ConcurrentHashMap<>(ratedRequests.size());
 
         Map scriptsWithoutParams = new HashMap<>();
-        for (Entry entry : qualityTask.getTemplates().entrySet()) {
-            scriptsWithoutParams.put(entry.getKey(),
-                    scriptService.compile(entry.getValue(), TemplateScript.CONTEXT));
+        for (Entry entry : evaluationSpecification.getTemplates().entrySet()) {
+            scriptsWithoutParams.put(entry.getKey(), scriptService.compile(entry.getValue(), TemplateScript.CONTEXT));
         }
 
         for (RatedRequest ratedRequest : ratedRequests) {
-            final RankEvalActionListener searchListener = new RankEvalActionListener(listener,
-                    qualityTask.getMetric(), ratedRequest, partialResults, errors, responseCounter);
             SearchSourceBuilder ratedSearchSource = ratedRequest.getTestRequest();
             if (ratedSearchSource == null) {
                 Map params = ratedRequest.getParams();
@@ -116,34 +112,32 @@ public class TransportRankEvalAction
                     listener.onFailure(e);
                 }
             }
+
             List summaryFields = ratedRequest.getSummaryFields();
             if (summaryFields.isEmpty()) {
                 ratedSearchSource.fetchSource(false);
             } else {
-                ratedSearchSource.fetchSource(
-                        summaryFields.toArray(new String[summaryFields.size()]), new String[0]);
+                ratedSearchSource.fetchSource(summaryFields.toArray(new String[summaryFields.size()]), new String[0]);
             }
 
-            String[] indices = new String[ratedRequest.getIndices().size()];
-            indices = ratedRequest.getIndices().toArray(indices);
-            SearchRequest templatedRequest = new SearchRequest(indices, ratedSearchSource);
-
+            SearchRequest templatedRequest = new SearchRequest(indices.toArray(new String[indices.size()]), ratedSearchSource);
+            final RankEvalActionListener searchListener = new RankEvalActionListener(listener,
+                    evaluationSpecification.getMetric(), ratedRequest, partialResults, errors, responseCounter);
             RequestTask task = new RequestTask(templatedRequest, searchListener);
             taskQueue.add(task);
         }
 
-        // Execute top n tasks, further execution is triggered in
-        // RankEvalActionListener
+        // Execute top n tasks, further execution is triggered in RankEvalActionListener
         for (int i = 0; (i < Math.min(ratedRequests.size(),
-                qualityTask.getMaxConcurrentSearches())); i++) {
+                evaluationSpecification.getMaxConcurrentSearches())); i++) {
             RequestTask task = taskQueue.poll();
             client.search(task.request, task.searchListener);
         }
     }
 
     private class RequestTask {
-        public SearchRequest request;
-        public RankEvalActionListener searchListener;
+        private SearchRequest request;
+        private RankEvalActionListener searchListener;
 
         RequestTask(SearchRequest request, RankEvalActionListener listener) {
             this.request = request;
@@ -151,7 +145,7 @@ public class TransportRankEvalAction
         }
     }
 
-    public class RankEvalActionListener implements ActionListener {
+    class RankEvalActionListener implements ActionListener {
 
         private ActionListener listener;
         private RatedRequest specification;
@@ -160,7 +154,7 @@ public class TransportRankEvalAction
         private EvaluationMetric metric;
         private AtomicInteger responseCounter;
 
-        public RankEvalActionListener(ActionListener listener,
+        RankEvalActionListener(ActionListener listener,
                 EvaluationMetric metric, RatedRequest specification,
                 Map details, Map errors,
                 AtomicInteger responseCounter) {
@@ -189,9 +183,7 @@ public class TransportRankEvalAction
 
         private void handleResponse() {
             if (responseCounter.decrementAndGet() == 0) {
-                // TODO add other statistics like micro/macro avg?
-                listener.onResponse(new RankEvalResponse(metric.combine(requestDetails.values()),
-                        requestDetails, errors));
+                listener.onResponse(new RankEvalResponse(metric.combine(requestDetails.values()), requestDetails, errors));
             } else {
                 if (!taskQueue.isEmpty()) {
                     RequestTask task = taskQueue.poll();
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
index b6bd80f497e..f3c38b7ae64 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
@@ -40,7 +40,6 @@ import java.util.List;
 import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments;
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 
-
 public class DiscountedCumulativeGainTests extends ESTestCase {
 
     /**
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java
index fb1c7db554a..26ff5f3683e 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java
@@ -47,8 +47,11 @@ public class EvalQueryQualityTests extends ESTestCase {
         EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAlphaOfLength(10),
                 randomDoubleBetween(0.0, 1.0, true));
         if (randomBoolean()) {
-            // TODO randomize this
-            evalQueryQuality.addMetricDetails(new PrecisionAtK.Breakdown(1, 5));
+            if (randomBoolean()) {
+                evalQueryQuality.setMetricDetails(new PrecisionAtK.Breakdown(randomIntBetween(0, 1000), randomIntBetween(0, 1000)));
+            } else {
+                evalQueryQuality.setMetricDetails(new MeanReciprocalRank.Breakdown(randomIntBetween(0, 1000)));
+            }
         }
         evalQueryQuality.addHitsAndRatings(ratedHits);
         return evalQueryQuality;
@@ -96,7 +99,7 @@ public class EvalQueryQualityTests extends ESTestCase {
             throw new IllegalStateException("The test should only allow four parameters mutated");
         }
         EvalQueryQuality evalQueryQuality = new EvalQueryQuality(id, qualityLevel);
-        evalQueryQuality.addMetricDetails(metricDetails);
+        evalQueryQuality.setMetricDetails(metricDetails);
         evalQueryQuality.addHitsAndRatings(ratedHits);
         return evalQueryQuality;
     }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
index 3b674d7e8a6..707a1448c81 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
@@ -70,19 +70,18 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         testQuery.query(new MatchAllQueryBuilder());
         RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query",
                 createRelevant("2", "3", "4", "5"), testQuery);
-        amsterdamRequest.setIndices(indices);
-        amsterdamRequest.setSummaryFields(Arrays.asList(new String[] { "text", "title" }));
+        amsterdamRequest.addSummaryFields(Arrays.asList(new String[] { "text", "title" }));
 
         specifications.add(amsterdamRequest);
         RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("1"),
                 testQuery);
-        berlinRequest.setIndices(indices);
-        berlinRequest.setSummaryFields(Arrays.asList(new String[] { "text", "title" }));
+        berlinRequest.addSummaryFields(Arrays.asList(new String[] { "text", "title" }));
 
         specifications.add(berlinRequest);
 
         PrecisionAtK metric = new PrecisionAtK(1, true);
         RankEvalSpec task = new RankEvalSpec(specifications, metric);
+        task.addIndices(indices);
 
         RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(),
                 RankEvalAction.INSTANCE, new RankEvalRequest());
@@ -90,7 +89,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
 
         RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request())
                 .actionGet();
-        assertEquals(1.0, response.getQualityLevel(), Double.MIN_VALUE);
+        assertEquals(1.0, response.getEvaluationResult(), Double.MIN_VALUE);
         Set> entrySet = response.getPartialResults().entrySet();
         assertEquals(2, entrySet.size());
         for (Entry entry : entrySet) {
@@ -136,17 +135,16 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         amsterdamQuery.query(new MatchAllQueryBuilder());
         RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query",
                 createRelevant("2", "3", "4", "5"), amsterdamQuery);
-        amsterdamRequest.setIndices(indices);
         specifications.add(amsterdamRequest);
 
         SearchSourceBuilder brokenQuery = new SearchSourceBuilder();
         brokenQuery.query(QueryBuilders.termQuery("population", "noStringOnNumericFields"));
         RatedRequest brokenRequest = new RatedRequest("broken_query", createRelevant("1"),
                 brokenQuery);
-        brokenRequest.setIndices(indices);
         specifications.add(brokenRequest);
 
         RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK());
+        task.addIndices(indices);
 
         RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(),
                 RankEvalAction.INSTANCE, new RankEvalRequest());
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
index aff758c096b..e1264c02f67 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
@@ -66,7 +66,7 @@ public class RankEvalResponseTests extends ESTestCase {
             try (StreamInput in = output.bytes().streamInput()) {
                 RankEvalResponse deserializedResponse = new RankEvalResponse();
                 deserializedResponse.readFrom(in);
-                assertEquals(randomResponse.getQualityLevel(), deserializedResponse.getQualityLevel(), Double.MIN_VALUE);
+                assertEquals(randomResponse.getEvaluationResult(), deserializedResponse.getEvaluationResult(), Double.MIN_VALUE);
                 assertEquals(randomResponse.getPartialResults(), deserializedResponse.getPartialResults());
                 assertEquals(randomResponse.getFailures().keySet(), deserializedResponse.getFailures().keySet());
                 assertNotSame(randomResponse, deserializedResponse);
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java
index 6197efb1b04..30d9291e134 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java
@@ -94,6 +94,12 @@ public class RankEvalSpecTests extends ESTestCase {
         }
         RankEvalSpec spec = new RankEvalSpec(ratedRequests, metric, templates);
         maybeSet(spec::setMaxConcurrentSearches, randomInt(100));
+        List indices = new ArrayList<>();
+        int size = randomIntBetween(0, 20);
+        for (int i = 0; i < size; i++) {
+            indices.add(randomAlphaOfLengthBetween(0, 50));
+        }
+        spec.addIndices(indices);
         return spec;
     }
 
@@ -137,11 +143,12 @@ public class RankEvalSpecTests extends ESTestCase {
         List ratedRequests = new ArrayList<>(original.getRatedRequests());
         EvaluationMetric metric = original.getMetric();
         Map templates = original.getTemplates();
+        List indices = new ArrayList<>(original.getIndices());
 
-        int mutate = randomIntBetween(0, 2);
+        int mutate = randomIntBetween(0, 3);
         switch (mutate) {
         case 0:
-            RatedRequest request = RatedRequestsTests.createTestItem(new ArrayList<>(), true);
+            RatedRequest request = RatedRequestsTests.createTestItem(true);
             ratedRequests.add(request);
             break;
         case 1:
@@ -154,6 +161,9 @@ public class RankEvalSpecTests extends ESTestCase {
         case 2:
             templates.put("mutation", new Script(ScriptType.INLINE, "mustache", randomAlphaOfLength(10), new HashMap<>()));
             break;
+        case 3:
+            indices.add(randomAlphaOfLength(5));
+            break;
         default:
             throw new IllegalStateException("Requested to modify more than available parameters.");
         }
@@ -162,31 +172,28 @@ public class RankEvalSpecTests extends ESTestCase {
         for (Entry entry : templates.entrySet()) {
             scripts.add(new ScriptWithId(entry.getKey(), entry.getValue()));
         }
-
         RankEvalSpec result = new RankEvalSpec(ratedRequests, metric, scripts);
+        result.addIndices(indices);
         return result;
     }
 
-    public void testMissingRatedRequestsFailsParsing() {
+    public void testMissingRatedRequestsFails() {
         EvaluationMetric metric = new PrecisionAtK();
-        expectThrows(IllegalStateException.class, () -> new RankEvalSpec(new ArrayList<>(), metric));
-        expectThrows(IllegalStateException.class, () -> new RankEvalSpec(null, metric));
+        expectThrows(IllegalArgumentException.class, () -> new RankEvalSpec(new ArrayList<>(), metric));
+        expectThrows(IllegalArgumentException.class, () -> new RankEvalSpec(null, metric));
     }
 
-    public void testMissingMetricFailsParsing() {
-        List strings = Arrays.asList("value");
-        List ratedRequests = randomList(() -> RatedRequestsTests.createTestItem(strings, randomBoolean()));
-        expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, null));
+    public void testMissingMetricFails() {
+        List ratedRequests = randomList(() -> RatedRequestsTests.createTestItem(randomBoolean()));
+        expectThrows(NullPointerException.class, () -> new RankEvalSpec(ratedRequests, null));
     }
 
-    public void testMissingTemplateAndSearchRequestFailsParsing() {
+    public void testMissingTemplateAndSearchRequestFails() {
         List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
         Map params = new HashMap<>();
         params.put("key", "value");
-
         RatedRequest request = new RatedRequest("id", ratedDocs, params, "templateId");
         List ratedRequests = Arrays.asList(request);
-
         expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, new PrecisionAtK()));
     }
 }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java
index 53846406d9c..a4ec29f9209 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java
@@ -73,7 +73,7 @@ public class RatedRequestsTests extends ESTestCase {
         return xContentRegistry;
     }
 
-    public static RatedRequest createTestItem(List indices, boolean forceRequest) {
+    public static RatedRequest createTestItem(boolean forceRequest) {
         String requestId = randomAlphaOfLength(50);
 
         List ratedDocs = new ArrayList<>();
@@ -104,31 +104,22 @@ public class RatedRequestsTests extends ESTestCase {
         RatedRequest ratedRequest = null;
         if (params.size() == 0) {
             ratedRequest = new RatedRequest(requestId, ratedDocs, testRequest);
-            ratedRequest.setIndices(indices);
-            ratedRequest.setSummaryFields(summaryFields);
+            ratedRequest.addSummaryFields(summaryFields);
         } else {
             ratedRequest = new RatedRequest(requestId, ratedDocs, params, randomAlphaOfLength(5));
-            ratedRequest.setIndices(indices);
-            ratedRequest.setSummaryFields(summaryFields);
+            ratedRequest.addSummaryFields(summaryFields);
         }
         return ratedRequest;
     }
 
     public void testXContentRoundtrip() throws IOException {
-        List indices = new ArrayList<>();
-        int size = randomIntBetween(0, 20);
-        for (int i = 0; i < size; i++) {
-            indices.add(randomAlphaOfLengthBetween(0, 50));
-        }
-
-        RatedRequest testItem = createTestItem(indices, randomBoolean());
+        RatedRequest testItem = createTestItem(randomBoolean());
         XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
         XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
         try (XContentParser itemParser = createParser(shuffled)) {
             itemParser.nextToken();
 
             RatedRequest parsedItem = RatedRequest.fromXContent(itemParser);
-            parsedItem.setIndices(indices); // IRL these come from URL parameters - see RestRankEvalAction
             assertNotSame(testItem, parsedItem);
             assertEquals(testItem, parsedItem);
             assertEquals(testItem.hashCode(), parsedItem.hashCode());
@@ -136,12 +127,7 @@ public class RatedRequestsTests extends ESTestCase {
     }
 
     public void testSerialization() throws IOException {
-        List indices = new ArrayList<>();
-        int size = randomIntBetween(0, 20);
-        for (int i = 0; i < size; i++) {
-            indices.add(randomAlphaOfLengthBetween(0, 50));
-        }
-        RatedRequest original = createTestItem(indices, randomBoolean());
+        RatedRequest original = createTestItem(randomBoolean());
         RatedRequest deserialized = copy(original);
         assertEquals(deserialized, original);
         assertEquals(deserialized.hashCode(), original.hashCode());
@@ -155,24 +141,18 @@ public class RatedRequestsTests extends ESTestCase {
     }
 
     public void testEqualsAndHash() throws IOException {
-        List indices = new ArrayList<>();
-        int size = randomIntBetween(0, 20);
-        for (int i = 0; i < size; i++) {
-            indices.add(randomAlphaOfLengthBetween(0, 50));
-        }
-        checkEqualsAndHashCode(createTestItem(indices, randomBoolean()), RatedRequestsTests::copy, RatedRequestsTests::mutateTestItem);
+        checkEqualsAndHashCode(createTestItem(randomBoolean()), RatedRequestsTests::copy, RatedRequestsTests::mutateTestItem);
     }
 
     private static RatedRequest mutateTestItem(RatedRequest original) {
         String id = original.getId();
         SearchSourceBuilder testRequest = original.getTestRequest();
         List ratedDocs = original.getRatedDocs();
-        List indices = original.getIndices();
         Map params = original.getParams();
         List summaryFields = original.getSummaryFields();
         String templateId = original.getTemplateId();
 
-        int mutate = randomIntBetween(0, 4);
+        int mutate = randomIntBetween(0, 3);
         switch (mutate) {
         case 0:
             id = randomValueOtherThan(id, () -> randomAlphaOfLength(10));
@@ -198,79 +178,65 @@ public class RatedRequestsTests extends ESTestCase {
             ratedDocs = Arrays.asList(randomValueOtherThanMany(ratedDocs::contains, () -> RatedDocumentTests.createRatedDocument()));
             break;
         case 3:
-            indices = Arrays.asList(randomValueOtherThanMany(indices::contains, () -> randomAlphaOfLength(10)));
-            break;
-        case 4:
             summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains, () -> randomAlphaOfLength(10)));
             break;
         default:
             throw new IllegalStateException("Requested to modify more than available parameters.");
         }
 
-        RatedRequest ratedRequest = new RatedRequest(id, ratedDocs, testRequest, params, templateId);
-        ratedRequest.setIndices(indices);
-        ratedRequest.setSummaryFields(summaryFields);
+        RatedRequest ratedRequest;
+        if (testRequest == null) {
+            ratedRequest = new RatedRequest(id, ratedDocs, params, templateId);
+        } else {
+            ratedRequest = new RatedRequest(id, ratedDocs, testRequest);
+        }
+        ratedRequest.addSummaryFields(summaryFields);
 
         return ratedRequest;
     }
 
     public void testDuplicateRatedDocThrowsException() {
         List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1), new RatedDocument("index1", "id1", 5));
-
-        // search request set, no summary fields
         IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder()));
-        assertEquals("Found duplicate rated document key [{\"_index\":\"index1\",\"_id\":\"id1\"}]",
+                () -> new RatedRequest("test_query", ratedDocs, new SearchSourceBuilder()));
+        assertEquals("Found duplicate rated document key [{\"_index\":\"index1\",\"_id\":\"id1\"}] in evaluation request [test_query]",
                 ex.getMessage());
-        // templated path, no summary fields
         Map params = new HashMap<>();
         params.put("key", "value");
-        ex = expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, params, "templateId"));
-        assertEquals("Found duplicate rated document key [{\"_index\":\"index1\",\"_id\":\"id1\"}]",
+        ex = expectThrows(IllegalArgumentException.class, () -> new RatedRequest("test_query", ratedDocs, params, "templateId"));
+        assertEquals("Found duplicate rated document key [{\"_index\":\"index1\",\"_id\":\"id1\"}] in evaluation request [test_query]",
                 ex.getMessage());
     }
 
     public void testNullSummaryFieldsTreatment() {
         List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
         RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder());
-        expectThrows(IllegalArgumentException.class, () -> request.setSummaryFields(null));
+        expectThrows(NullPointerException.class, () -> request.addSummaryFields(null));
     }
 
     public void testNullParamsTreatment() {
         List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
-        RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, null);
+        RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder());
         assertNotNull(request.getParams());
-    }
-
-    public void testSettingParamsAndRequestThrows() {
-        List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
-        Map params = new HashMap<>();
-        params.put("key", "value");
-        expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), params, null));
+        assertEquals(0, request.getParams().size());
     }
 
     public void testSettingNeitherParamsNorRequestThrows() {
         List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
         expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null));
-        expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, new HashMap<>(), "templateId"));
+        expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, new HashMap<>(), "templateId"));
     }
 
     public void testSettingParamsWithoutTemplateIdThrows() {
         List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
         Map params = new HashMap<>();
         params.put("key", "value");
-        expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, params, null));
-    }
-
-    public void testSettingTemplateIdAndRequestThrows() {
-        List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
-        expectThrows(IllegalArgumentException.class,
-                () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, "templateId"));
+        expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, params, null));
     }
 
     public void testSettingTemplateIdNoParamsThrows() {
         List ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
-        expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null, "templateId"));
+        expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, "templateId"));
     }
 
     /**
diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/index/rankeval/SmokeMultipleTemplatesIT.java
similarity index 85%
rename from qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java
rename to qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/index/rankeval/SmokeMultipleTemplatesIT.java
index d4a0411349d..04d5d94023b 100644
--- a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeMultipleTemplatesIT.java
+++ b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/index/rankeval/SmokeMultipleTemplatesIT.java
@@ -17,18 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.smoketest;
+package org.elasticsearch.index.rankeval;
 
-import org.elasticsearch.index.rankeval.Precision;
-import org.elasticsearch.index.rankeval.RankEvalAction;
-import org.elasticsearch.index.rankeval.RankEvalPlugin;
-import org.elasticsearch.index.rankeval.RankEvalRequest;
-import org.elasticsearch.index.rankeval.RankEvalRequestBuilder;
-import org.elasticsearch.index.rankeval.RankEvalResponse;
-import org.elasticsearch.index.rankeval.RankEvalSpec;
 import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId;
-import org.elasticsearch.index.rankeval.RatedDocument;
-import org.elasticsearch.index.rankeval.RatedRequest;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -88,7 +79,6 @@ public class SmokeMultipleTemplatesIT  extends ESIntegTestCase {
         ams_params.put("querystring", "amsterdam");
         RatedRequest amsterdamRequest = new RatedRequest(
                 "amsterdam_query", createRelevant("2", "3", "4", "5"), ams_params, MATCH_TEMPLATE);
-        amsterdamRequest.setIndices(indices);
 
         specifications.add(amsterdamRequest);
 
@@ -96,10 +86,9 @@ public class SmokeMultipleTemplatesIT  extends ESIntegTestCase {
         berlin_params.put("querystring", "berlin");
         RatedRequest berlinRequest = new RatedRequest(
                 "berlin_query", createRelevant("1"), berlin_params, MATCH_TEMPLATE);
-        berlinRequest.setIndices(indices);
         specifications.add(berlinRequest);
 
-        Precision metric = new Precision();
+        PrecisionAtK metric = new PrecisionAtK();
 
         ScriptWithId template =
                 new ScriptWithId(
@@ -111,11 +100,12 @@ public class SmokeMultipleTemplatesIT  extends ESIntegTestCase {
         Set templates = new HashSet<>();
         templates.add(template);
         RankEvalSpec task = new RankEvalSpec(specifications, metric, templates);
+        task.addIndices(indices);
         RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
         builder.setRankEvalSpec(task);
 
         RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
-        assertEquals(0.9, response.getQualityLevel(), Double.MIN_VALUE);
+        assertEquals(0.9, response.getEvaluationResult(), Double.MIN_VALUE);
     }
 
     private static List createRelevant(String... docs) {
diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/index/rankeval/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java
similarity index 97%
rename from qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java
rename to qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/index/rankeval/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java
index 8132b5851e2..b8b1607065c 100644
--- a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java
+++ b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/index/rankeval/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.smoketest;
+package org.elasticsearch.index.rankeval;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;

From fdb24cd3e4357b72863a29d1a63dd581936ba142 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= 
Date: Wed, 15 Nov 2017 18:35:52 +0100
Subject: [PATCH 111/297] Fixing occasional test failure in RankEvalSpecTests

---
 .../java/org/elasticsearch/index/rankeval/RankEvalSpec.java | 5 +++--
 .../org/elasticsearch/index/rankeval/RankEvalSpecTests.java | 6 ++++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java
index cae326ad1a2..2d3842daf70 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java
@@ -262,11 +262,12 @@ public class RankEvalSpec implements Writeable, ToXContentObject {
         return Objects.equals(ratedRequests, other.ratedRequests) &&
                 Objects.equals(metric, other.metric) &&
                 Objects.equals(maxConcurrentSearches, other.maxConcurrentSearches) &&
-                Objects.equals(templates, other.templates);
+                Objects.equals(templates, other.templates) &&
+                Objects.equals(indices, other.indices);
     }
 
     @Override
     public final int hashCode() {
-        return Objects.hash(ratedRequests, metric, templates, maxConcurrentSearches);
+        return Objects.hash(ratedRequests, metric, templates, maxConcurrentSearches, indices);
     }
 }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java
index 30d9291e134..5f1f6cfc0b9 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java
@@ -110,7 +110,9 @@ public class RankEvalSpecTests extends ESTestCase {
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, shuffled.bytes())) {
 
             RankEvalSpec parsedItem = RankEvalSpec.parse(parser);
-            // IRL these come from URL parameters - see RestRankEvalAction
+            // indices, come from URL parameters, so they don't survive xContent roundtrip
+            // for the sake of being able to use equals() next, we add it to the parsed object
+            parsedItem.addIndices(testItem.getIndices());
             assertNotSame(testItem, parsedItem);
             assertEquals(testItem, parsedItem);
             assertEquals(testItem.hashCode(), parsedItem.hashCode());
@@ -142,7 +144,7 @@ public class RankEvalSpecTests extends ESTestCase {
     private static RankEvalSpec mutateTestItem(RankEvalSpec original) {
         List ratedRequests = new ArrayList<>(original.getRatedRequests());
         EvaluationMetric metric = original.getMetric();
-        Map templates = original.getTemplates();
+        Map templates = new HashMap<>(original.getTemplates());
         List indices = new ArrayList<>(original.getIndices());
 
         int mutate = randomIntBetween(0, 3);

From 504462f61777926d972feff9e5a2cbd1c8d7c41c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= 
Date: Wed, 15 Nov 2017 19:00:37 +0100
Subject: [PATCH 112/297] Reverting change to Script in core, can go in
 separately

---
 core/src/main/java/org/elasticsearch/script/Script.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/script/Script.java b/core/src/main/java/org/elasticsearch/script/Script.java
index f2eddbd65f6..7e931a89467 100644
--- a/core/src/main/java/org/elasticsearch/script/Script.java
+++ b/core/src/main/java/org/elasticsearch/script/Script.java
@@ -448,7 +448,7 @@ public final class Script implements ToXContentObject, Writeable {
             this.lang = in.readOptionalString();
             this.idOrCode = in.readString();
             @SuppressWarnings("unchecked")
-            Map options = (Map)in.readMap();
+            Map options = (Map)(Map)in.readMap();
             this.options = options;
             this.params = in.readMap();
         // Version 5.1 to 5.3 (exclusive) requires all Script members to be non-null and supports the potential
@@ -461,7 +461,7 @@ public final class Script implements ToXContentObject, Writeable {
 
             this.idOrCode = in.readString();
             @SuppressWarnings("unchecked")
-            Map options = (Map)in.readMap();
+            Map options = (Map)(Map)in.readMap();
 
             if (this.type != ScriptType.INLINE && options.isEmpty()) {
                 this.options = null;
@@ -518,7 +518,7 @@ public final class Script implements ToXContentObject, Writeable {
             out.writeOptionalString(lang);
             out.writeString(idOrCode);
             @SuppressWarnings("unchecked")
-            Map options = (Map)this.options;
+            Map options = (Map)(Map)this.options;
             out.writeMap(options);
             out.writeMap(params);
         // Version 5.1 to 5.3 (exclusive) requires all Script members to be non-null and supports the potential
@@ -535,7 +535,7 @@ public final class Script implements ToXContentObject, Writeable {
 
             out.writeString(idOrCode);
             @SuppressWarnings("unchecked")
-            Map options = (Map)this.options;
+            Map options = (Map)(Map)this.options;
 
             if (options == null) {
                 out.writeMap(new HashMap<>());

From 35fabdaf8ac6df3f7b55021182778e7e3bf5f1c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= 
Date: Thu, 16 Nov 2017 14:06:10 +0100
Subject: [PATCH 113/297] Parse EvluationMetrics as named Objects

---
 .../index/rankeval/EvaluationMetric.java      | 33 -------------------
 .../index/rankeval/RankEvalPlugin.java        | 14 ++++++++
 .../index/rankeval/RankEvalSpec.java          | 26 +++++++--------
 .../rankeval/MeanReciprocalRankTests.java     |  2 +-
 .../index/rankeval/RankEvalSpecTests.java     | 20 ++++++-----
 .../index/rankeval/RatedRequestsTests.java    |  3 --
 6 files changed, 40 insertions(+), 58 deletions(-)

diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
index 865093e9a9e..444666ee0d4 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
@@ -19,16 +19,12 @@
 
 package org.elasticsearch.index.rankeval;
 
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -58,35 +54,6 @@ public interface EvaluationMetric extends ToXContent, NamedWriteable {
      */
     EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs);
 
-    static EvaluationMetric fromXContent(XContentParser parser) throws IOException {
-        EvaluationMetric rc;
-        Token token = parser.nextToken();
-        if (token != XContentParser.Token.FIELD_NAME) {
-            throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name");
-        }
-        String metricName = parser.currentName();
-
-        // TODO switch to using a plugable registry
-        switch (metricName) {
-        case PrecisionAtK.NAME:
-            rc = PrecisionAtK.fromXContent(parser);
-            break;
-        case MeanReciprocalRank.NAME:
-            rc = MeanReciprocalRank.fromXContent(parser);
-            break;
-        case DiscountedCumulativeGain.NAME:
-            rc = DiscountedCumulativeGain.fromXContent(parser);
-            break;
-        default:
-            throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName);
-        }
-        if (parser.currentToken() == XContentParser.Token.END_OBJECT) {
-            // if we are at END_OBJECT, move to the next one...
-            parser.nextToken();
-        }
-        return rc;
-    }
-
     /**
      * join hits with rated documents using the joint _index/_id document key
      */
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java
index 53c976392ce..278b7f1a613 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java
@@ -23,11 +23,13 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
@@ -64,4 +66,16 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin {
                 .add(new NamedWriteableRegistry.Entry(MetricDetails.class, MeanReciprocalRank.NAME, MeanReciprocalRank.Breakdown::new));
         return namedWriteables;
     }
+
+    @Override
+    public List getNamedXContent() {
+        List namedXContent = new ArrayList<>();
+        namedXContent.add(
+                new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(PrecisionAtK.NAME), PrecisionAtK::fromXContent));
+        namedXContent.add(new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(MeanReciprocalRank.NAME),
+                MeanReciprocalRank::fromXContent));
+        namedXContent.add(new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(DiscountedCumulativeGain.NAME),
+                DiscountedCumulativeGain::fromXContent));
+        return namedXContent;
+    }
 }
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java
index 2d3842daf70..8b396d0b862 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.script.Script;
 
 import java.io.IOException;
@@ -169,22 +170,21 @@ public class RankEvalSpec implements Writeable, ToXContentObject {
             a -> new RankEvalSpec((List) a[0], (EvaluationMetric) a[1], (Collection) a[2]));
 
     static {
-        PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> {
-            return RatedRequest.fromXContent(p);
-        }, REQUESTS_FIELD);
-        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> {
-            try {
-                return EvaluationMetric.fromXContent(p);
-            } catch (IOException ex) {
-                throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex);
-            }
-        }, METRIC_FIELD);
-        PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
-            return ScriptWithId.fromXContent(p);
-        }, TEMPLATES_FIELD);
+        PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> RatedRequest.fromXContent(p), REQUESTS_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> parseMetric(p), METRIC_FIELD);
+        PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> ScriptWithId.fromXContent(p),
+                TEMPLATES_FIELD);
         PARSER.declareInt(RankEvalSpec::setMaxConcurrentSearches, MAX_CONCURRENT_SEARCHES_FIELD);
     }
 
+    private static EvaluationMetric parseMetric(XContentParser parser) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+        EvaluationMetric metric = parser.namedObject(EvaluationMetric.class, parser.currentName(), null);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        return metric;
+    }
+
     public static RankEvalSpec parse(XContentParser parser) {
         return PARSER.apply(parser, null);
     }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java
index 81586783c06..3a1b4939758 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java
@@ -166,7 +166,7 @@ public class MeanReciprocalRankTests extends ESTestCase {
         return hits;
     }
 
-    private static MeanReciprocalRank createTestItem() {
+    static MeanReciprocalRank createTestItem() {
         return new MeanReciprocalRank(randomIntBetween(0, 20));
     }
 
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java
index 5f1f6cfc0b9..f3385514da7 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.rankeval;
 
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -49,6 +50,12 @@ import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashC
 
 public class RankEvalSpecTests extends ESTestCase {
 
+    @SuppressWarnings("resource")
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(new RankEvalPlugin().getNamedXContent());
+    }
+
     private static  List randomList(Supplier randomSupplier) {
         List result = new ArrayList<>();
         int size = randomIntBetween(1, 20);
@@ -59,12 +66,10 @@ public class RankEvalSpecTests extends ESTestCase {
     }
 
     private static RankEvalSpec createTestItem() throws IOException {
-        EvaluationMetric metric;
-        if (randomBoolean()) {
-            metric = PrecisionAtKTests.createTestItem();
-        } else {
-            metric = DiscountedCumulativeGainTests.createTestItem();
-        }
+        Supplier metric = randomFrom(Arrays.asList(
+                () -> PrecisionAtKTests.createTestItem(),
+                () -> MeanReciprocalRankTests.createTestItem(),
+                () -> DiscountedCumulativeGainTests.createTestItem()));
 
         List ratedRequests = null;
         Collection templates = null;
@@ -92,7 +97,7 @@ public class RankEvalSpecTests extends ESTestCase {
                     new SearchSourceBuilder());
             ratedRequests = Arrays.asList(ratedRequest);
         }
-        RankEvalSpec spec = new RankEvalSpec(ratedRequests, metric, templates);
+        RankEvalSpec spec = new RankEvalSpec(ratedRequests, metric.get(), templates);
         maybeSet(spec::setMaxConcurrentSearches, randomInt(100));
         List indices = new ArrayList<>();
         int size = randomIntBetween(0, 20);
@@ -105,7 +110,6 @@ public class RankEvalSpecTests extends ESTestCase {
 
     public void testXContentRoundtrip() throws IOException {
         RankEvalSpec testItem = createTestItem();
-
         XContentBuilder shuffled = shuffleXContent(testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS));
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, shuffled.bytes())) {
 
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java
index a4ec29f9209..e46d253dd4d 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java
@@ -53,9 +53,6 @@ public class RatedRequestsTests extends ESTestCase {
 
     private static NamedXContentRegistry xContentRegistry;
 
-    /**
-     * setup for the whole base test class
-     */
     @BeforeClass
     public static void init() {
         xContentRegistry = new NamedXContentRegistry(

From 94a0631a3e714fd34e073a6a1a9d9df9cf465402 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= 
Date: Thu, 16 Nov 2017 18:09:16 +0100
Subject: [PATCH 114/297] [Tests] Add testToXContent() RankEvalResponseTests

---
 .../index/rankeval/EvalQueryQuality.java      | 18 +++---
 .../index/rankeval/RankEvalResponseTests.java | 60 +++++++++++++++----
 2 files changed, 57 insertions(+), 21 deletions(-)

diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java
index a42ce60c4e4..28162c47441 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java
@@ -39,7 +39,7 @@ public class EvalQueryQuality implements ToXContent, Writeable {
     private final String queryId;
     private final double evaluationResult;
     private MetricDetails optionalMetricDetails;
-    private final List hits = new ArrayList<>();
+    private final List ratedHits = new ArrayList<>();
 
     public EvalQueryQuality(String id, double evaluationResult) {
         this.queryId = id;
@@ -48,7 +48,7 @@ public class EvalQueryQuality implements ToXContent, Writeable {
 
     public EvalQueryQuality(StreamInput in) throws IOException {
         this(in.readString(), in.readDouble());
-        this.hits.addAll(in.readList(RatedSearchHit::new));
+        this.ratedHits.addAll(in.readList(RatedSearchHit::new));
         this.optionalMetricDetails = in.readOptionalNamedWriteable(MetricDetails.class);
     }
 
@@ -56,7 +56,7 @@ public class EvalQueryQuality implements ToXContent, Writeable {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(queryId);
         out.writeDouble(evaluationResult);
-        out.writeList(hits);
+        out.writeList(ratedHits);
         out.writeOptionalNamedWriteable(this.optionalMetricDetails);
     }
 
@@ -77,11 +77,11 @@ public class EvalQueryQuality implements ToXContent, Writeable {
     }
 
     public void addHitsAndRatings(List hits) {
-        this.hits.addAll(hits);
+        this.ratedHits.addAll(hits);
     }
 
     public List getHitsAndRatings() {
-        return this.hits;
+        return this.ratedHits;
     }
 
     @Override
@@ -89,7 +89,7 @@ public class EvalQueryQuality implements ToXContent, Writeable {
         builder.startObject(queryId);
         builder.field("quality_level", this.evaluationResult);
         builder.startArray("unknown_docs");
-        for (DocumentKey key : EvaluationMetric.filterUnknownDocuments(hits)) {
+        for (DocumentKey key : EvaluationMetric.filterUnknownDocuments(ratedHits)) {
             builder.startObject();
             builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), key.getIndex());
             builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), key.getDocId());
@@ -97,7 +97,7 @@ public class EvalQueryQuality implements ToXContent, Writeable {
         }
         builder.endArray();
         builder.startArray("hits");
-        for (RatedSearchHit hit : hits) {
+        for (RatedSearchHit hit : ratedHits) {
             hit.toXContent(builder, params);
         }
         builder.endArray();
@@ -121,12 +121,12 @@ public class EvalQueryQuality implements ToXContent, Writeable {
         EvalQueryQuality other = (EvalQueryQuality) obj;
         return Objects.equals(queryId, other.queryId) &&
                 Objects.equals(evaluationResult, other.evaluationResult) &&
-                Objects.equals(hits, other.hits) &&
+                Objects.equals(ratedHits, other.ratedHits) &&
                 Objects.equals(optionalMetricDetails, other.optionalMetricDetails);
     }
 
     @Override
     public final int hashCode() {
-        return Objects.hash(queryId, evaluationResult, hits, optionalMetricDetails);
+        return Objects.hash(queryId, evaluationResult, ratedHits, optionalMetricDetails);
     }
 }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
index e1264c02f67..827f7be4442 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
@@ -19,20 +19,28 @@
 
 package org.elasticsearch.index.rankeval;
 
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 public class RankEvalResponseTests extends ESTestCase {
 
@@ -41,13 +49,14 @@ public class RankEvalResponseTests extends ESTestCase {
         Map partials = new HashMap<>(numberOfRequests);
         for (int i = 0; i < numberOfRequests; i++) {
             String id = randomAlphaOfLengthBetween(3, 10);
-            int numberOfUnknownDocs = randomIntBetween(0, 5);
-            List unknownDocs = new ArrayList<>(numberOfUnknownDocs);
-            for (int d = 0; d < numberOfUnknownDocs; d++) {
-                unknownDocs.add(new DocumentKey(randomAlphaOfLength(10), randomAlphaOfLength(10)));
-            }
             EvalQueryQuality evalQuality = new EvalQueryQuality(id,
                     randomDoubleBetween(0.0, 1.0, true));
+            int numberOfDocs = randomIntBetween(0, 5);
+            List ratedHits = new ArrayList<>(numberOfDocs);
+            for (int d = 0; d < numberOfDocs; d++) {
+                ratedHits.add(searchHit(randomAlphaOfLength(10), randomIntBetween(0, 1000), randomIntBetween(0, 10)));
+            }
+            evalQuality.addHitsAndRatings(ratedHits);
             partials.put(id, evalQuality);
         }
         int numberOfErrors = randomIntBetween(0, 2);
@@ -76,12 +85,39 @@ public class RankEvalResponseTests extends ESTestCase {
     }
 
     public void testToXContent() throws IOException {
-        RankEvalResponse randomResponse = createRandomResponse();
+        EvalQueryQuality coffeeQueryQuality = new EvalQueryQuality("coffee_query", 0.1);
+        coffeeQueryQuality.addHitsAndRatings(Arrays.asList(searchHit("index", 123, 5), searchHit("index", 456, null)));
+        RankEvalResponse response = new RankEvalResponse(0.123, Collections.singletonMap("coffee_query", coffeeQueryQuality),
+                Collections.singletonMap("beer_query", new ParsingException(new XContentLocation(0, 0), "someMsg")));
         XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        if (ESTestCase.randomBoolean()) {
-            builder.prettyPrint();
-        }
-        randomResponse.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        // TODO check the correctness of the output
+        String xContent = response.toXContent(builder, ToXContent.EMPTY_PARAMS).bytes().utf8ToString();
+        assertEquals(("{" +
+                "    \"rank_eval\": {" +
+                "        \"quality_level\": 0.123," +
+                "        \"details\": {" +
+                "            \"coffee_query\": {" +
+                "                \"quality_level\": 0.1," +
+                "                \"unknown_docs\": [{\"_index\":\"index\",\"_id\":\"456\"}]," +
+                "                \"hits\":[{\"hit\":{\"_index\":\"index\",\"_type\":\"\",\"_id\":\"123\",\"_score\":1.0}," +
+                "                           \"rating\":5}," +
+                "                          {\"hit\":{\"_index\":\"index\",\"_type\":\"\",\"_id\":\"456\",\"_score\":1.0}," +
+                "                           \"rating\":null}" +
+                "                         ]" +
+                "            }" +
+                "        }," +
+                "        \"failures\": {" +
+                "            \"beer_query\": {" +
+                "                \"error\": \"ParsingException[someMsg]\"" +
+                "            }" +
+                "        }" +
+                "    }" +
+                "}").replaceAll("\\s+", ""), xContent);
+    }
+
+    private static RatedSearchHit searchHit(String index, int docId, Integer rating) {
+        SearchHit hit = new SearchHit(docId, docId + "", new Text(""), Collections.emptyMap());
+        hit.shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null));
+        hit.score(1.0f);
+        return new RatedSearchHit(hit, rating != null ? Optional.of(rating) : Optional.empty());
     }
 }

From 57354772835b7a5665f8feb3637146cd1039097b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?=
 
Date: Thu, 23 Nov 2017 12:31:25 +0100
Subject: [PATCH 115/297] Fix some documentation typos

---
 docs/reference/search/rank-eval.asciidoc | 25 ++++++++++++------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc
index 2ca1441db4a..0e9e8a821ac 100644
--- a/docs/reference/search/rank-eval.asciidoc
+++ b/docs/reference/search/rank-eval.asciidoc
@@ -17,11 +17,11 @@ They usually enters some search terms into a search box or some other web form.
 All of this information, together with meta information about the user (e.g. the browser, location, earlier preferences etc...) then gets translated into a query to the underlying search system.
 
 The challenge for search engineers is to tweak this translation process from user entries to a concrete query in such a way, that the search results contain the most relevant information with respect to the users information_need.
-This can only be done if the search result quality is evaluated constantly across a representative test suit of typical user queries, so that improvements in the rankings for one particular query doesn't negatively effect the ranking for other types of queries.
+This can only be done if the search result quality is evaluated constantly across a representative test suite of typical user queries, so that improvements in the rankings for one particular query doesn't negatively effect the ranking for other types of queries.
 
-In order to get started with search quality evaluation, three basic things a needed:
+In order to get started with search quality evaluation, three basic things are needed:
 
-. a collection of document you want to evaluate your query performance against, usually one or more indices
+. a collection of documents you want to evaluate your query performance against, usually one or more indices
 . a collection of typical search requests that users enter into your system
 . a set of document ratings that judge the documents relevance with respect to a search request+
   It is important to note that one set of document ratings is needed per test query, and that
@@ -45,7 +45,7 @@ GET /my_index/_rank_eval
 ------------------------------
 // NOTCONSOLE
 
-<1> a set of typical search requests to your system
+<1> a set of typical search requests, together with their provided ratings
 <2> definition of the evaluation metric to calculate
 <3> a specific metric and its parameters
 
@@ -131,16 +131,15 @@ GET /my_index/_rank_eval
 The `metric` section determines which of the available evaluation metrics is going to be used.
 Currently, the following metrics are supported:
 
-==== Precision at k (Prec@k)
+==== Precision at K (P@k)
 
 This metric measures the number of relevant results in the top k search results. Its a form of the well known https://en.wikipedia.org/wiki/Information_retrieval#Precision[Precision] metric that only looks at the top k documents. It is the fraction of relevant documents in those first k
-search. A precision at 10 (prec@10) value of 0.6 then means six out of the 10 top hits where
-relevant with respect to the users information need.
+search. A precision at 10 (P@10) value of 0.6 then means six out of the 10 top hits are relevant with respect to the users information need.
 
-This metric works well as a first and an easy to explain evaluation metric. 
-Documents in the collection need to be rated either as relevant or irrelevant with respect to the current query. Prec@k does not take into account where in the top k results the relevant documents occur, so a ranking of ten results that contains one relevant
-result in position 10 is equally good as a ranking of ten results that contains
-one relevant result in position 1.
+P@k works well as a simple evaluation metric that has the benefit of being easy to understand and explain.
+Documents in the collection need to be rated either as relevant or irrelevant with respect to the current query. 
+P@k does not take into account where in the top k results the relevant documents occur, so a ranking of ten results that 
+contains one relevant result in position 10 is equally good as a ranking of ten results that contains one relevant result in position 1.
 
 [source,js]
 --------------------------------
@@ -168,7 +167,7 @@ The `precision` metric takes the following optional parameters
 [cols="<,<",options="header",]
 |=======================================================================
 |Parameter |Description
-|`relevant_rating_threshold` |Sets the rating threshold from which on documents are considered to be
+|`relevant_rating_threshold` |Sets the rating threshold above which documents are considered to be
 "relevant". Defaults to `1`.
 |`ignore_unlabeled` |controls how unlabeled documents in the search results are counted.
 If set to 'true', unlabeled documents are ignored and neither count as relevant or irrelevant. Set to 'false' (the default), they are treated as irrelevant.
@@ -206,7 +205,7 @@ The `mean_reciprocal_rank` metric takes the following optional parameters
 [cols="<,<",options="header",]
 |=======================================================================
 |Parameter |Description
-|`relevant_rating_threshold` |Sets the rating threshold from which on documents are considered to be
+|`relevant_rating_threshold` |Sets the rating threshold above which documents are considered to be
 "relevant". Defaults to `1`.
 |=======================================================================
 

From 1352b7c6ea2c2e1eeca2f776065853bb6a8ee27e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?=
 <10398885+cbuescher@users.noreply.github.com>
Date: Mon, 27 Nov 2017 10:15:59 +0100
Subject: [PATCH 116/297] Use msearch instead of single search (#27520)

Change TransportRankEvalAction to use one MultiSearchRequest instead of issuing several parallel search requests to simplify the transport action.
---
 .../rankeval/TransportRankEvalAction.java     | 106 +++++++-----------
 1 file changed, 40 insertions(+), 66 deletions(-)

diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
index 658cdbd0d68..0ff491cb8bc 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
@@ -20,8 +20,10 @@
 package org.elasticsearch.index.rankeval;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.MultiSearchRequest;
+import org.elasticsearch.action.search.MultiSearchResponse;
+import org.elasticsearch.action.search.MultiSearchResponse.Item;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.Client;
@@ -41,15 +43,12 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.Collection;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Queue;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.elasticsearch.common.xcontent.XContentHelper.createParser;
 
@@ -69,7 +68,6 @@ import static org.elasticsearch.common.xcontent.XContentHelper.createParser;
 public class TransportRankEvalAction extends HandledTransportAction {
     private Client client;
     private ScriptService scriptService;
-    Queue taskQueue = new ConcurrentLinkedQueue<>();
     private NamedXContentRegistry namedXContentRegistry;
 
     @Inject
@@ -88,10 +86,7 @@ public class TransportRankEvalAction extends HandledTransportAction indices = evaluationSpecification.getIndices();
 
-        Collection ratedRequests = evaluationSpecification.getRatedRequests();
-        AtomicInteger responseCounter = new AtomicInteger(ratedRequests.size());
-        Map partialResults = new ConcurrentHashMap<>(
-                ratedRequests.size());
+        List ratedRequests = evaluationSpecification.getRatedRequests();
         Map errors = new ConcurrentHashMap<>(ratedRequests.size());
 
         Map scriptsWithoutParams = new HashMap<>();
@@ -99,6 +94,9 @@ public class TransportRankEvalAction extends HandledTransportAction ratedRequestsInSearch = new ArrayList<>();
         for (RatedRequest ratedRequest : ratedRequests) {
             SearchSourceBuilder ratedSearchSource = ratedRequest.getTestRequest();
             if (ratedSearchSource == null) {
@@ -109,87 +107,63 @@ public class TransportRankEvalAction extends HandledTransportAction summaryFields = ratedRequest.getSummaryFields();
             if (summaryFields.isEmpty()) {
                 ratedSearchSource.fetchSource(false);
             } else {
                 ratedSearchSource.fetchSource(summaryFields.toArray(new String[summaryFields.size()]), new String[0]);
             }
-
-            SearchRequest templatedRequest = new SearchRequest(indices.toArray(new String[indices.size()]), ratedSearchSource);
-            final RankEvalActionListener searchListener = new RankEvalActionListener(listener,
-                    evaluationSpecification.getMetric(), ratedRequest, partialResults, errors, responseCounter);
-            RequestTask task = new RequestTask(templatedRequest, searchListener);
-            taskQueue.add(task);
-        }
-
-        // Execute top n tasks, further execution is triggered in RankEvalActionListener
-        for (int i = 0; (i < Math.min(ratedRequests.size(),
-                evaluationSpecification.getMaxConcurrentSearches())); i++) {
-            RequestTask task = taskQueue.poll();
-            client.search(task.request, task.searchListener);
+            msearchRequest.add(new SearchRequest(indices.toArray(new String[indices.size()]), ratedSearchSource));
         }
+        assert ratedRequestsInSearch.size() == msearchRequest.requests().size();
+        client.multiSearch(msearchRequest, new RankEvalActionListener(listener, evaluationSpecification.getMetric(),
+                ratedRequestsInSearch.toArray(new RatedRequest[ratedRequestsInSearch.size()]), errors));
     }
 
-    private class RequestTask {
-        private SearchRequest request;
-        private RankEvalActionListener searchListener;
+    class RankEvalActionListener implements ActionListener {
 
-        RequestTask(SearchRequest request, RankEvalActionListener listener) {
-            this.request = request;
-            this.searchListener = listener;
-        }
-    }
+        private final ActionListener listener;
+        private final RatedRequest[] specifications;
 
-    class RankEvalActionListener implements ActionListener {
+        private final Map errors;
+        private final EvaluationMetric metric;
 
-        private ActionListener listener;
-        private RatedRequest specification;
-        private Map requestDetails;
-        private Map errors;
-        private EvaluationMetric metric;
-        private AtomicInteger responseCounter;
-
-        RankEvalActionListener(ActionListener listener,
-                EvaluationMetric metric, RatedRequest specification,
-                Map details, Map errors,
-                AtomicInteger responseCounter) {
+        RankEvalActionListener(ActionListener listener, EvaluationMetric metric, RatedRequest[] specifications,
+                Map errors) {
             this.listener = listener;
             this.metric = metric;
             this.errors = errors;
-            this.specification = specification;
-            this.requestDetails = details;
-            this.responseCounter = responseCounter;
+            this.specifications = specifications;
         }
 
         @Override
-        public void onResponse(SearchResponse searchResponse) {
-            SearchHit[] hits = searchResponse.getHits().getHits();
-            EvalQueryQuality queryQuality = metric.evaluate(specification.getId(), hits,
-                    specification.getRatedDocs());
-            requestDetails.put(specification.getId(), queryQuality);
-            handleResponse();
+        public void onResponse(MultiSearchResponse multiSearchResponse) {
+            int responsePosition = 0;
+            Map responseDetails = new HashMap<>(specifications.length);
+            for (Item response : multiSearchResponse.getResponses()) {
+                RatedRequest specification = specifications[responsePosition];
+                if (response.isFailure() == false) {
+                    SearchHit[] hits = response.getResponse().getHits().getHits();
+                    EvalQueryQuality queryQuality = this.metric.evaluate(specification.getId(), hits, specification.getRatedDocs());
+                    responseDetails.put(specification.getId(), queryQuality);
+                } else {
+                    errors.put(specification.getId(), response.getFailure());
+                }
+                responsePosition++;
+            }
+            listener.onResponse(new RankEvalResponse(this.metric.combine(responseDetails.values()), responseDetails, this.errors));
         }
 
         @Override
         public void onFailure(Exception exception) {
-            errors.put(specification.getId(), exception);
-            handleResponse();
-        }
-
-        private void handleResponse() {
-            if (responseCounter.decrementAndGet() == 0) {
-                listener.onResponse(new RankEvalResponse(metric.combine(requestDetails.values()), requestDetails, errors));
-            } else {
-                if (!taskQueue.isEmpty()) {
-                    RequestTask task = taskQueue.poll();
-                    client.search(task.request, task.searchListener);
-                }
-            }
+            listener.onFailure(exception);
         }
     }
 }

From d0cd18169e0cd1d9301c5c46a9f874abfc822d31 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Tue, 28 Nov 2017 22:43:34 -0500
Subject: [PATCH 117/297] Remove stale awaits fix on azure master nodes test

This awaits fix has been there forever and no one seems to know what to
do with this test. I say let CI churn on it because it passed for me
three out of three times. If there is something wrong with it, we will
know quickly and can then address with the new information that we have.
---
 .../discovery/azure/classic/AzureMinimumMasterNodesTests.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
index 5300981c4ef..f639c064d37 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
@@ -41,7 +41,6 @@ import static org.hamcrest.Matchers.nullValue;
         transportClientRatio = 0.0,
         numClientNodes = 0,
         autoMinMasterNodes = false)
-@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/27554")
 public class AzureMinimumMasterNodesTests extends AbstractAzureComputeServiceTestCase {
 
     public AzureMinimumMasterNodesTests() {

From 547f0061188044bc742f0a78b12d62cbcfb52ffd Mon Sep 17 00:00:00 2001
From: Tanguy Leroux 
Date: Wed, 29 Nov 2017 09:39:49 +0100
Subject: [PATCH 118/297] Remove XContentType auto detection in
 BlobStoreRepository (#27480)

---
 .../repositories/blobstore/BlobStoreRepository.java           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 84d3d743f64..440867577ea 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -616,7 +616,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                 BytesStreamOutput out = new BytesStreamOutput();
                 Streams.copy(blob, out);
                 // EMPTY is safe here because RepositoryData#fromXContent calls namedObject
-                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
+                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes(), XContentType.JSON)) {
                     repositoryData = RepositoryData.snapshotsFromXContent(parser, indexGen);
                 } catch (NotXContentException e) {
                     logger.warn("[{}] index blob is not valid x-content [{} bytes]", snapshotsIndexBlobName, out.bytes().length());
@@ -628,7 +628,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             try (InputStream blob = snapshotsBlobContainer.readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) {
                 BytesStreamOutput out = new BytesStreamOutput();
                 Streams.copy(blob, out);
-                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
+                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes(), XContentType.JSON)) {
                     repositoryData = repositoryData.incompatibleSnapshotsFromXContent(parser);
                 }
             } catch (NoSuchFileException e) {

From 0d11b9fe34c007fddc5cbf2c821ef226b7a0fc75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?=
 <10398885+cbuescher@users.noreply.github.com>
Date: Wed, 29 Nov 2017 09:44:25 +0100
Subject: [PATCH 119/297] [Docs] Unify spelling of Elasticsearch (#27567)

Removes occurences of "elasticsearch" or "ElasticSearch" in favour of
"Elasticsearch" where appropriate.
---
 docs/community-clients/index.asciidoc         |  4 ++--
 docs/groovy-api/anatomy.asciidoc              |  2 +-
 docs/groovy-api/client.asciidoc               |  6 +++---
 docs/groovy-api/index.asciidoc                |  2 +-
 docs/java-api/client.asciidoc                 |  4 ++--
 docs/java-api/docs/bulk.asciidoc              |  4 ++--
 docs/java-api/index.asciidoc                  |  4 ++--
 docs/java-api/indexed-scripts.asciidoc        |  2 +-
 docs/java-rest/low-level/sniffer.asciidoc     |  6 +++---
 docs/java-rest/low-level/usage.asciidoc       |  6 +++---
 docs/plugins/authors.asciidoc                 |  6 +++---
 docs/plugins/discovery-azure-classic.asciidoc | 16 +++++++--------
 docs/plugins/discovery-ec2.asciidoc           |  4 ++--
 docs/plugins/discovery-gce.asciidoc           | 12 +++++------
 docs/plugins/integrations.asciidoc            | 10 +++++-----
 docs/plugins/repository-azure.asciidoc        |  2 +-
 docs/plugins/repository-gcs.asciidoc          |  4 ++--
 docs/plugins/repository-hdfs.asciidoc         |  2 +-
 docs/plugins/repository-s3.asciidoc           |  4 ++--
 .../bucket/datehistogram-aggregation.asciidoc |  2 +-
 .../significantterms-aggregation.asciidoc     |  2 +-
 .../bucket/terms-aggregation.asciidoc         |  6 +++---
 .../metrics/cardinality-aggregation.asciidoc  |  2 +-
 docs/reference/api-conventions.asciidoc       |  6 +++---
 docs/reference/cat/indices.asciidoc           |  2 +-
 docs/reference/docs/index_.asciidoc           |  4 ++--
 docs/reference/docs/reindex.asciidoc          |  2 +-
 docs/reference/glossary.asciidoc              | 10 +++++-----
 docs/reference/how-to/disk-usage.asciidoc     |  8 ++++----
 docs/reference/how-to/indexing-speed.asciidoc |  4 ++--
 docs/reference/how-to/search-speed.asciidoc   |  8 ++++----
 .../index-modules/index-sorting.asciidoc      |  4 ++--
 docs/reference/index-modules/merge.asciidoc   |  2 +-
 docs/reference/index-modules/store.asciidoc   |  6 +++---
 .../mapping/dynamic/templates.asciidoc        |  8 ++++----
 docs/reference/modules/discovery/zen.asciidoc |  2 +-
 docs/reference/modules/http.asciidoc          |  4 ++--
 docs/reference/modules/memcached.asciidoc     |  4 ++--
 docs/reference/modules/plugins.asciidoc       |  2 +-
 docs/reference/modules/snapshots.asciidoc     |  4 ++--
 docs/reference/modules/thrift.asciidoc        |  2 +-
 docs/reference/modules/transport.asciidoc     |  2 +-
 docs/reference/search/profile.asciidoc        |  2 +-
 docs/reference/search/request/sort.asciidoc   |  4 ++--
 .../reference/search/search-template.asciidoc |  2 +-
 docs/reference/search/search.asciidoc         |  4 ++--
 .../setup/install/sysconfig-file.asciidoc     |  4 ++--
 docs/reference/setup/secure-settings.asciidoc |  4 ++--
 docs/reference/testing.asciidoc               |  2 +-
 .../testing/testing-framework.asciidoc        | 20 +++++++++----------
 50 files changed, 119 insertions(+), 119 deletions(-)

diff --git a/docs/community-clients/index.asciidoc b/docs/community-clients/index.asciidoc
index 5da117cc7cf..ca37c42fb02 100644
--- a/docs/community-clients/index.asciidoc
+++ b/docs/community-clients/index.asciidoc
@@ -129,7 +129,7 @@ The following project appears to be abandoned:
 == Lua
 
 * https://github.com/DhavalKapil/elasticsearch-lua[elasticsearch-lua]:
-  Lua client for elasticsearch
+  Lua client for Elasticsearch
 
 [[dotnet]]
 == .NET
@@ -218,7 +218,7 @@ Also see the {client}/ruby-api/current/index.html[official Elasticsearch Ruby cl
   Tiny client with built-in zero-downtime migrations and ActiveRecord integration.
 
 * https://github.com/toptal/chewy[chewy]:
-  Chewy is ODM and wrapper for official elasticsearch client
+  Chewy is an ODM and wrapper for the official Elasticsearch client
 
 * https://github.com/ankane/searchkick[Searchkick]:
   Intelligent search made easy
diff --git a/docs/groovy-api/anatomy.asciidoc b/docs/groovy-api/anatomy.asciidoc
index 99e008fb6eb..2da39a05ba5 100644
--- a/docs/groovy-api/anatomy.asciidoc
+++ b/docs/groovy-api/anatomy.asciidoc
@@ -13,7 +13,7 @@ The first type is to simply provide the request as a Closure, which
 automatically gets resolved into the respective request instance (for
 the index API, its the `IndexRequest` class). The API returns a special
 future, called `GActionFuture`. This is a groovier version of
-elasticsearch Java `ActionFuture` (in turn a nicer extension to Java own
+Elasticsearch Java `ActionFuture` (in turn a nicer extension to Java own
 `Future`) which allows to register listeners (closures) on it for
 success and failures, as well as blocking for the response. For example:
 
diff --git a/docs/groovy-api/client.asciidoc b/docs/groovy-api/client.asciidoc
index a2745f459bd..c3c89e71bc5 100644
--- a/docs/groovy-api/client.asciidoc
+++ b/docs/groovy-api/client.asciidoc
@@ -1,7 +1,7 @@
 [[client]]
 == Client
 
-Obtaining an elasticsearch Groovy `GClient` (a `GClient` is a simple
+Obtaining an Elasticsearch Groovy `GClient` (a `GClient` is a simple
 wrapper on top of the Java `Client`) is simple. The most common way to
 get a client is by starting an embedded `Node` which acts as a node
 within the cluster.
@@ -11,7 +11,7 @@ within the cluster.
 === Node Client
 
 A Node based client is the simplest form to get a `GClient` to start
-executing operations against elasticsearch.
+executing operations against Elasticsearch.
 
 [source,groovy]
 --------------------------------------------------
@@ -29,7 +29,7 @@ GClient client = node.client();
 node.close();
 --------------------------------------------------
 
-Since elasticsearch allows to configure it using JSON based settings,
+Since Elasticsearch allows to configure it using JSON based settings,
 the configuration itself can be done using a closure that represent the
 JSON:
 
diff --git a/docs/groovy-api/index.asciidoc b/docs/groovy-api/index.asciidoc
index dd819c07c1e..e1bb81856f1 100644
--- a/docs/groovy-api/index.asciidoc
+++ b/docs/groovy-api/index.asciidoc
@@ -6,7 +6,7 @@ include::../Versions.asciidoc[]
 == Preface
 
 This section describes the http://groovy-lang.org/[Groovy] API
-elasticsearch provides. All elasticsearch APIs are executed using a
+Elasticsearch provides. All Elasticsearch APIs are executed using a
 <>, and are completely
 asynchronous in nature (they either accept a listener, or return a
 future).
diff --git a/docs/java-api/client.asciidoc b/docs/java-api/client.asciidoc
index 83889bda49a..d5893ffce95 100644
--- a/docs/java-api/client.asciidoc
+++ b/docs/java-api/client.asciidoc
@@ -8,7 +8,7 @@ You can use the *Java client* in multiple ways:
   existing cluster
 * Perform administrative tasks on a running cluster
 
-Obtaining an elasticsearch `Client` is simple. The most common way to
+Obtaining an Elasticsearch `Client` is simple. The most common way to
 get a client is by creating a <>
 that connects to a cluster.
 
@@ -69,7 +69,7 @@ After this, the client will call the internal cluster state API on those nodes
 to discover available data nodes. The internal node list of the client will
 be replaced with those data nodes only. This list is refreshed every five seconds by default.
 Note that the IP addresses the sniffer connects to are the ones declared as the 'publish'
-address in those node's elasticsearch config.
+address in those node's Elasticsearch config.
 
 Keep in mind that the list might possibly not include the original node it connected to
 if that node is not a data node. If, for instance, you initially connect to a
diff --git a/docs/java-api/docs/bulk.asciidoc b/docs/java-api/docs/bulk.asciidoc
index 07849164a68..57f55b5171a 100644
--- a/docs/java-api/docs/bulk.asciidoc
+++ b/docs/java-api/docs/bulk.asciidoc
@@ -78,7 +78,7 @@ BulkProcessor bulkProcessor = BulkProcessor.builder(
             BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3)) <9>
         .build();
 --------------------------------------------------
-<1> Add your elasticsearch client
+<1> Add your Elasticsearch client
 <2> This method is called just before bulk is executed. You can for example see the numberOfActions with
     `request.numberOfActions()`
 <3> This method is called after bulk execution. You can for example check if there was some failing requests
@@ -138,7 +138,7 @@ all bulk requests to complete then returns `true`, if the specified waiting time
 [[java-docs-bulk-processor-tests]]
 ==== Using Bulk Processor in tests
 
-If you are running tests with elasticsearch and are using the `BulkProcessor` to populate your dataset
+If you are running tests with Elasticsearch and are using the `BulkProcessor` to populate your dataset
 you should better set the number of concurrent requests to `0` so the flush operation of the bulk will be executed
 in a synchronous manner:
 
diff --git a/docs/java-api/index.asciidoc b/docs/java-api/index.asciidoc
index 10430adc9a6..75e97f8a45a 100644
--- a/docs/java-api/index.asciidoc
+++ b/docs/java-api/index.asciidoc
@@ -5,8 +5,8 @@ include::../Versions.asciidoc[]
 
 [preface]
 == Preface
-This section describes the Java API that elasticsearch provides. All
-elasticsearch operations are executed using a
+This section describes the Java API that Elasticsearch provides. All
+Elasticsearch operations are executed using a
 <> object. All
 operations are completely asynchronous in nature (either accepts a
 listener, or returns a future).
diff --git a/docs/java-api/indexed-scripts.asciidoc b/docs/java-api/indexed-scripts.asciidoc
index a1259649a77..f1877bba1f8 100644
--- a/docs/java-api/indexed-scripts.asciidoc
+++ b/docs/java-api/indexed-scripts.asciidoc
@@ -2,7 +2,7 @@
 == Indexed Scripts API
 
 The indexed script API allows one to interact with scripts and templates
-stored in an elasticsearch index. It can be used to create, update, get, 
+stored in an Elasticsearch index. It can be used to create, update, get, 
 and delete indexed scripts and templates.
 
 [source,java]
diff --git a/docs/java-rest/low-level/sniffer.asciidoc b/docs/java-rest/low-level/sniffer.asciidoc
index 4af61b5d2f3..df772643bf4 100644
--- a/docs/java-rest/low-level/sniffer.asciidoc
+++ b/docs/java-rest/low-level/sniffer.asciidoc
@@ -16,10 +16,10 @@ The javadoc for the REST client sniffer can be found at {rest-client-sniffer-jav
 === Maven Repository
 
 The REST client sniffer is subject to the same release cycle as
-elasticsearch. Replace the version with the desired sniffer version, first
+Elasticsearch. Replace the version with the desired sniffer version, first
 released with `5.0.0-alpha4`. There is no relation between the sniffer version
-and the elasticsearch version that the client can communicate with. Sniffer
-supports fetching the nodes list from elasticsearch 2.x and onwards.
+and the Elasticsearch version that the client can communicate with. Sniffer
+supports fetching the nodes list from Elasticsearch 2.x and onwards.
 
 
 ==== Maven configuration
diff --git a/docs/java-rest/low-level/usage.asciidoc b/docs/java-rest/low-level/usage.asciidoc
index d39fab38dda..6e25c506acb 100644
--- a/docs/java-rest/low-level/usage.asciidoc
+++ b/docs/java-rest/low-level/usage.asciidoc
@@ -17,10 +17,10 @@ http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.elasticsearch.client%22[Ma
 Central]. The minimum Java version required is `1.7`.
 
 The low-level REST client is subject to the same release cycle as
-elasticsearch. Replace the version with the desired client version, first
+Elasticsearch. Replace the version with the desired client version, first
 released with `5.0.0-alpha4`. There is no relation between the client version
-and the elasticsearch version that the client can communicate with. The
-low-level REST client is compatible with all elasticsearch versions.
+and the Elasticsearch version that the client can communicate with. The
+low-level REST client is compatible with all Elasticsearch versions.
 
 [[java-rest-low-usage-maven-maven]]
 ==== Maven configuration
diff --git a/docs/plugins/authors.asciidoc b/docs/plugins/authors.asciidoc
index 3b91fb76f66..8dc06d1433a 100644
--- a/docs/plugins/authors.asciidoc
+++ b/docs/plugins/authors.asciidoc
@@ -46,7 +46,7 @@ can fill in the necessary values in the `build.gradle` file for your plugin.
 Use the system property `java.specification.version`. Version string must be a sequence
 of nonnegative decimal integers separated by "."'s and may have leading zeros.
 
-|`elasticsearch.version`    |String  | version of elasticsearch compiled against.
+|`elasticsearch.version`    |String  | version of Elasticsearch compiled against.
 
 |=======================================================================
 
@@ -57,7 +57,7 @@ If you need other resources, package them into a resources jar.
 .Plugin release lifecycle
 ==============================================
 
-You will have to release a new version of the plugin for each new elasticsearch release.
+You will have to release a new version of the plugin for each new Elasticsearch release.
 This version is checked when the plugin is loaded so Elasticsearch will refuse to start
 in the presence of plugins with the incorrect `elasticsearch.version`.
 
@@ -86,7 +86,7 @@ with a large warning, and they will have to confirm them when installing the
 plugin interactively. So if possible, it is best to avoid requesting any
 spurious permissions!
 
-If you are using the elasticsearch Gradle build system, place this file in
+If you are using the Elasticsearch Gradle build system, place this file in
 `src/main/plugin-metadata` and it will be applied during unit tests as well.
 
 Keep in mind that the Java security model is stack-based, and the additional
diff --git a/docs/plugins/discovery-azure-classic.asciidoc b/docs/plugins/discovery-azure-classic.asciidoc
index 9fd1f97129b..f11b4018bf5 100644
--- a/docs/plugins/discovery-azure-classic.asciidoc
+++ b/docs/plugins/discovery-azure-classic.asciidoc
@@ -37,7 +37,7 @@ discovery:
 .Binding the network host
 ==============================================
 
-The keystore file must be placed in a directory accessible by elasticsearch like the `config` directory.
+The keystore file must be placed in a directory accessible by Elasticsearch like the `config` directory.
 
 It's important to define `network.host` as by default it's bound to `localhost`.
 
@@ -95,7 +95,7 @@ The following are a list of settings that can further control the discovery:
 `discovery.azure.endpoint.name`::
 
     When using `public_ip` this setting is used to identify the endpoint name
-    used to forward requests to elasticsearch (aka transport port name).
+    used to forward requests to Elasticsearch (aka transport port name).
     Defaults to `elasticsearch`. In Azure management console, you could define
     an endpoint `elasticsearch` forwarding for example requests on public IP
     on port 8100 to the virtual machine on port 9300.
@@ -131,7 +131,7 @@ discovery:
 We will expose here one strategy which is to hide our Elasticsearch cluster from outside.
 
 With this strategy, only VMs behind the same virtual port can talk to each
-other.  That means that with this mode, you can use elasticsearch unicast
+other.  That means that with this mode, you can use Elasticsearch unicast
 discovery to build a cluster, using the Azure API to retrieve information
 about your nodes.
 
@@ -177,7 +177,7 @@ cat azure-cert.pem azure-pk.pem > azure.pem.txt
 openssl pkcs12 -export -in azure.pem.txt -out azurekeystore.pkcs12 -name azure -noiter -nomaciter
 ----
 
-Upload the `azure-certificate.cer` file both in the elasticsearch Cloud Service (under `Manage Certificates`),
+Upload the `azure-certificate.cer` file both in the Elasticsearch Cloud Service (under `Manage Certificates`),
 and under `Settings -> Manage Certificates`.
 
 IMPORTANT: When prompted for a password, you need to enter a non empty one.
@@ -354,7 +354,7 @@ sudo dpkg -i elasticsearch-{version}.deb
 ----
 // NOTCONSOLE
 
-Check that elasticsearch is running:
+Check that Elasticsearch is running:
 
 [source,js]
 ----
@@ -393,11 +393,11 @@ This command should give you a JSON result:
 // So much s/// but at least we test that the layout is close to matching....
 
 [[discovery-azure-classic-long-plugin]]
-===== Install elasticsearch cloud azure plugin
+===== Install Elasticsearch cloud azure plugin
 
 [source,sh]
 ----
-# Stop elasticsearch
+# Stop Elasticsearch
 sudo service elasticsearch stop
 
 # Install the plugin
@@ -428,7 +428,7 @@ discovery:
 # path.data: /mnt/resource/elasticsearch/data
 ----
 
-Restart elasticsearch:
+Restart Elasticsearch:
 
 [source,sh]
 ----
diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc
index 60ccaba00ba..2e2bc9cf268 100644
--- a/docs/plugins/discovery-ec2.asciidoc
+++ b/docs/plugins/discovery-ec2.asciidoc
@@ -181,10 +181,10 @@ to use include the `discovery.ec2.tag.` prefix. For example, setting `discovery.
 filter instances with a tag key set to `stage`, and a value of `dev`. Several tags set will require all of those tags
 to be set for the instance to be included.
 
-One practical use for tag filtering is when an ec2 cluster contains many nodes that are not running elasticsearch. In
+One practical use for tag filtering is when an ec2 cluster contains many nodes that are not running Elasticsearch. In
 this case (particularly with high `discovery.zen.ping_timeout` values) there is a risk that a new node's discovery phase
 will end before it has found the cluster (which will result in it declaring itself master of a new cluster with the same
-name - highly undesirable). Tagging elasticsearch ec2 nodes and then filtering by that tag will resolve this issue.
+name - highly undesirable). Tagging Elasticsearch ec2 nodes and then filtering by that tag will resolve this issue.
 
 [[discovery-ec2-attributes]]
 ===== Automatic Node Attributes
diff --git a/docs/plugins/discovery-gce.asciidoc b/docs/plugins/discovery-gce.asciidoc
index 4b31158b694..c194434414e 100644
--- a/docs/plugins/discovery-gce.asciidoc
+++ b/docs/plugins/discovery-gce.asciidoc
@@ -201,7 +201,7 @@ sudo dpkg -i elasticsearch-2.0.0.deb
 --------------------------------------------------
 
 [[discovery-gce-usage-long-install-plugin]]
-===== Install elasticsearch discovery gce plugin
+===== Install Elasticsearch discovery gce plugin
 
 Install the plugin:
 
@@ -231,7 +231,7 @@ discovery:
 --------------------------------------------------
 
 
-Start elasticsearch:
+Start Elasticsearch:
 
 [source,sh]
 --------------------------------------------------
@@ -354,9 +354,9 @@ For example, setting `discovery.gce.tags` to `dev` will only filter instances ha
 set will require all of those tags to be set for the instance to be included.
 
 One practical use for tag filtering is when an GCE cluster contains many nodes that are not running
-elasticsearch. In this case (particularly with high `discovery.zen.ping_timeout` values) there is a risk that a new
+Elasticsearch. In this case (particularly with high `discovery.zen.ping_timeout` values) there is a risk that a new
 node's discovery phase will end before it has found the cluster (which will result in it declaring itself master of a
-new cluster with the same name - highly undesirable). Adding tag on elasticsearch GCE nodes and then filtering by that
+new cluster with the same name - highly undesirable). Adding tag on Elasticsearch GCE nodes and then filtering by that
 tag will resolve this issue.
 
 Add your tag when building the new instance:
@@ -385,8 +385,8 @@ discovery:
 [[discovery-gce-usage-port]]
 ==== Changing default transport port
 
-By default, elasticsearch GCE plugin assumes that you run elasticsearch on 9300 default port.
-But you can specify the port value elasticsearch is meant to use using google compute engine metadata `es_port`:
+By default, Elasticsearch GCE plugin assumes that you run Elasticsearch on 9300 default port.
+But you can specify the port value Elasticsearch is meant to use using google compute engine metadata `es_port`:
 
 [[discovery-gce-usage-port-create]]
 ===== When creating instance
diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc
index 9ece2febde1..162988fe3fc 100644
--- a/docs/plugins/integrations.asciidoc
+++ b/docs/plugins/integrations.asciidoc
@@ -60,10 +60,10 @@ releases 2.0 and later do not support rivers.
   The Java Database Connection (JDBC) importer allows to fetch data from JDBC sources for indexing into Elasticsearch (by Jörg Prante)
 
 * https://github.com/reachkrishnaraj/kafka-elasticsearch-standalone-consumer/tree/branch2.0[Kafka Standalone Consumer(Indexer)]:
-  Kafka Standalone Consumer [Indexer] will read messages from Kafka in batches, processes(as implemented) and bulk-indexes them into ElasticSearch. Flexible and scalable. More documentation in above GitHub repo's Wiki.(Please use branch 2.0)!
+  Kafka Standalone Consumer [Indexer] will read messages from Kafka in batches, processes(as implemented) and bulk-indexes them into Elasticsearch. Flexible and scalable. More documentation in above GitHub repo's Wiki.(Please use branch 2.0)!
 
 * https://github.com/ozlerhakan/mongolastic[Mongolastic]:
-  A tool that clones data from ElasticSearch to MongoDB and vice versa
+  A tool that clones data from Elasticsearch to MongoDB and vice versa
 
 * https://github.com/Aconex/scrutineer[Scrutineer]:
   A high performance consistency checker to compare what you've indexed
@@ -106,7 +106,7 @@ releases 2.0 and later do not support rivers.
   indexing in Elasticsearch.
 
 * https://camel.apache.org/elasticsearch.html[Apache Camel Integration]:
-  An Apache camel component to integrate elasticsearch
+  An Apache camel component to integrate Elasticsearch
 
 * https://metacpan.org/release/Catmandu-Store-ElasticSearch[Catmanadu]:
   An Elasticsearch backend for the Catmandu framework.
@@ -164,7 +164,7 @@ releases 2.0 and later do not support rivers.
   Nagios.
 
 * https://github.com/radu-gheorghe/check-es[check-es]:
-  Nagios/Shinken plugins for checking on elasticsearch
+  Nagios/Shinken plugins for checking on Elasticsearch
 
 * https://github.com/mattweber/es2graphite[es2graphite]:
   Send cluster and indices stats and status to Graphite for monitoring and graphing.
@@ -206,7 +206,7 @@ These projects appear to have been abandoned:
   Daikon Elasticsearch CLI
 
 * https://github.com/fullscale/dangle[dangle]:
-  A set of AngularJS directives that provide common visualizations for elasticsearch based on
+  A set of AngularJS directives that provide common visualizations for Elasticsearch based on
   D3.
 * https://github.com/OlegKunitsyn/eslogd[eslogd]:
   Linux daemon that replicates events to a central Elasticsearch server in realtime
diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc
index 583217324ed..9f80e4ab91c 100644
--- a/docs/plugins/repository-azure.asciidoc
+++ b/docs/plugins/repository-azure.asciidoc
@@ -34,7 +34,7 @@ bin/elasticsearch-keystore add azure.client.secondary.key
 `default` is the default account name which will be used by a repository unless you set an explicit one.
 
 You can set the client side timeout to use when making any single request. It can be defined globally, per account or both.
-It's not set by default which means that elasticsearch is using the
+It's not set by default which means that Elasticsearch is using the
 http://azure.github.io/azure-storage-java/com/microsoft/azure/storage/RequestOptions.html#setTimeoutIntervalInMs(java.lang.Integer)[default value]
 set by the azure client (known as 5 minutes).
 
diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc
index 90c6e8f0ba4..d7626d69090 100644
--- a/docs/plugins/repository-gcs.asciidoc
+++ b/docs/plugins/repository-gcs.asciidoc
@@ -42,7 +42,7 @@ The bucket should now be created.
 The plugin supports two authentication modes:
 
 * The built-in <>. This mode is
-recommended if your elasticsearch node is running on a Compute Engine virtual machine.
+recommended if your Elasticsearch node is running on a Compute Engine virtual machine.
 
 * Specifying <> credentials.
 
@@ -61,7 +61,7 @@ instance details at the section "Cloud API access scopes".
 
 [[repository-gcs-using-service-account]]
 ===== Using a Service Account
-If your elasticsearch node is not running on Compute Engine, or if you don't want to use Google's
+If your Elasticsearch node is not running on Compute Engine, or if you don't want to use Google's
 built-in authentication mechanism, you can authenticate on the Storage service using a
 https://cloud.google.com/iam/docs/overview#service_account[Service Account] file.
 
diff --git a/docs/plugins/repository-hdfs.asciidoc b/docs/plugins/repository-hdfs.asciidoc
index 933f6f69f62..ffd5ecebc25 100644
--- a/docs/plugins/repository-hdfs.asciidoc
+++ b/docs/plugins/repository-hdfs.asciidoc
@@ -95,7 +95,7 @@ methods are supported by the plugin:
 `simple`::
 
     Also means "no security" and is enabled by default. Uses information from underlying operating system account
-    running elasticsearch to inform Hadoop of the name of the current user. Hadoop makes no attempts to verify this
+    running Elasticsearch to inform Hadoop of the name of the current user. Hadoop makes no attempts to verify this
     information.
 
 `kerberos`::
diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index 565c94f5a7d..eb0828e96c6 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -284,7 +284,7 @@ You may further restrict the permissions by specifying a prefix within the bucke
 // NOTCONSOLE
 
 The bucket needs to exist to register a repository for snapshots. If you did not create the bucket then the repository
-registration will fail. If you want elasticsearch to create the bucket instead, you can add the permission to create a
+registration will fail. If you want Elasticsearch to create the bucket instead, you can add the permission to create a
 specific bucket like this:
 
 [source,js]
@@ -305,6 +305,6 @@ specific bucket like this:
 [float]
 ==== AWS VPC Bandwidth Settings
 
-AWS instances resolve S3 endpoints to a public IP. If the elasticsearch instances reside in a private subnet in an AWS VPC then all traffic to S3 will go through that VPC's NAT instance. If your VPC's NAT instance is a smaller instance size (e.g. a t1.micro) or is handling a high volume of network traffic your bandwidth to S3 may be limited by that NAT instance's networking bandwidth limitations.
+AWS instances resolve S3 endpoints to a public IP. If the Elasticsearch instances reside in a private subnet in an AWS VPC then all traffic to S3 will go through that VPC's NAT instance. If your VPC's NAT instance is a smaller instance size (e.g. a t1.micro) or is handling a high volume of network traffic your bandwidth to S3 may be limited by that NAT instance's networking bandwidth limitations.
 
 Instances residing in a public subnet in an AWS VPC will connect to S3 via the VPC's internet gateway and not be bandwidth limited by the VPC's NAT instance.
diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
index ea72e07e337..2cac8729b86 100644
--- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
@@ -2,7 +2,7 @@
 === Date Histogram Aggregation
 
 A multi-bucket aggregation similar to the <> except it can
-only be applied on date values. Since dates are represented in elasticsearch internally as long values, it is possible
+only be applied on date values. Since dates are represented in Elasticsearch internally as long values, it is possible
 to use the normal `histogram` on dates as well, though accuracy will be compromised. The reason for this is in the fact
 that time based intervals are not fixed (think of leap years and on the number of days in a month). For this reason,
 we need special support for time based data. From a functionality perspective, this histogram supports the same features
diff --git a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc
index 1db54611b31..2de6a0bbb2f 100644
--- a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc
@@ -449,7 +449,7 @@ a consolidated review by the reducing node before the final selection. Obviously
 will cause extra network traffic and RAM usage so this is  quality/cost trade off that needs to be balanced.  If `shard_size` is set to -1 (the default) then `shard_size` will be automatically estimated based on the number of shards and the `size` parameter.
 
 
-NOTE:   `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, elasticsearch will
+NOTE:   `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, Elasticsearch will
         override it and reset it to be equal to `size`.
 
 ===== Minimum document count
diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index ba6b912780c..e768cb0b295 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -99,7 +99,7 @@ Response:
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\.//]
 <1> an upper bound of the error on the document counts for each term, see <>
-<2> when there are lots of unique terms, elasticsearch only returns the top terms; this number is the sum of the document counts for all buckets that are not part of the response
+<2> when there are lots of unique terms, Elasticsearch only returns the top terms; this number is the sum of the document counts for all buckets that are not part of the response
 <3> the list of the top buckets, the meaning of `top` being defined by the <>
 
 By default, the `terms` aggregation will return the buckets for the top ten terms ordered by the `doc_count`. One can
@@ -210,7 +210,7 @@ one can increase the accuracy of the returned terms and avoid the overhead of st
 the client.
 
 
-NOTE:   `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, elasticsearch will
+NOTE:   `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, Elasticsearch will
         override it and reset it to be equal to `size`.
 
 
@@ -740,7 +740,7 @@ expire then we may be missing accounts of interest and have set our numbers too
 * increase the `size` parameter to return more results per partition (could be heavy on memory) or
 * increase the `num_partitions` to consider less accounts per request (could increase overall processing time as we need to make more requests)
 
-Ultimately this is a balancing act between managing the elasticsearch resources required to process a single request and the volume
+Ultimately this is a balancing act between managing the Elasticsearch resources required to process a single request and the volume
 of requests that the client application must issue to complete a task.
 
 ==== Multi-field terms aggregation
diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
index c861a0ef9b6..3e06c6f347f 100644
--- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
@@ -161,7 +161,7 @@ counting millions of items.
 On string fields that have a high cardinality, it might be faster to store the
 hash of your field values in your index and then run the cardinality aggregation
 on this field. This can either be done by providing hash values from client-side
-or by letting elasticsearch compute hash values for you by using the
+or by letting Elasticsearch compute hash values for you by using the
 {plugins}/mapper-murmur3.html[`mapper-murmur3`] plugin.
 
 NOTE: Pre-computing hashes is usually only useful on very large and/or
diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc
index 472c48e5232..3d8cba18e05 100644
--- a/docs/reference/api-conventions.asciidoc
+++ b/docs/reference/api-conventions.asciidoc
@@ -3,7 +3,7 @@
 
 [partintro]
 --
-The *elasticsearch* REST APIs are exposed using <>.
+The *Elasticsearch* REST APIs are exposed using <>.
 
 The conventions listed in this chapter can be applied throughout the REST
 API, unless otherwise specified.
@@ -228,7 +228,7 @@ Some examples are:
 === Response Filtering
 
 All REST APIs accept a `filter_path` parameter that can be used to reduce
-the response returned by elasticsearch. This parameter takes a comma
+the response returned by Elasticsearch. This parameter takes a comma
 separated list of filters expressed with the dot notation:
 
 [source,js]
@@ -360,7 +360,7 @@ Responds:
 --------------------------------------------------
 // TESTRESPONSE
 
-Note that elasticsearch sometimes returns directly the raw value of a field,
+Note that Elasticsearch sometimes returns directly the raw value of a field,
 like the `_source` field. If you want to filter `_source` fields, you should
 consider combining the already existing `_source` parameter (see
 <> for more details) with the `filter_path`
diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc
index 746d0b4bb58..3a50a836d0f 100644
--- a/docs/reference/cat/indices.asciidoc
+++ b/docs/reference/cat/indices.asciidoc
@@ -32,7 +32,7 @@ All these exposed metrics come directly from Lucene APIs.
 1. As the number of documents and deleted documents shown in this are at the lucene level,
 it includes all the hidden documents (e.g. from nested documents) as well.
 
-2. To get actual count of documents at the elasticsearch level, the recommended way
+2. To get actual count of documents at the Elasticsearch level, the recommended way
 is to use either the <> or the <>
 
 [float]
diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc
index 0ac58622d51..74d0ee157fd 100644
--- a/docs/reference/docs/index_.asciidoc
+++ b/docs/reference/docs/index_.asciidoc
@@ -133,7 +133,7 @@ version number is equal to zero.
 A nice side effect is that there is no need to maintain strict ordering
 of async indexing operations executed as a result of changes to a source
 database, as long as version numbers from the source database are used.
-Even the simple case of updating the elasticsearch index using data from
+Even the simple case of updating the Elasticsearch index using data from
 a database is simplified if external versioning is used, as only the
 latest version will be used if the index operations are out of order for
 whatever reason.
@@ -355,7 +355,7 @@ and isn't able to compare it against the new source.
 There isn't a hard and fast rule about when noop updates aren't acceptable.
 It's a combination of lots of factors like how frequently your data source
 sends updates that are actually noops and how many queries per second
-elasticsearch runs on the shard with receiving the updates.
+Elasticsearch runs on the shard with receiving the updates.
 
 [float]
 [[timeout]]
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index e1876327504..77ff6162dcf 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -1030,7 +1030,7 @@ PUT metricbeat-2016.05.31/beat/1?refresh
 ----------------------------------------------------------------
 // CONSOLE
 
-The new template for the `metricbeat-*` indices is already loaded into elasticsearch
+The new template for the `metricbeat-*` indices is already loaded into Elasticsearch
 but it applies only to the newly created indices. Painless can be used to reindex
 the existing documents and apply the new template.
 
diff --git a/docs/reference/glossary.asciidoc b/docs/reference/glossary.asciidoc
index 3559cb8985b..0012beebdca 100644
--- a/docs/reference/glossary.asciidoc
+++ b/docs/reference/glossary.asciidoc
@@ -16,7 +16,7 @@
   terms stored in the index.
   +
   It is this process of analysis (both at index time and at search time)
-  that allows elasticsearch to perform full text queries.
+  that allows Elasticsearch to perform full text queries.
   +
   Also see <> and <>.
 
@@ -29,7 +29,7 @@
 
 [[glossary-document]] document ::
 
-  A document is a JSON document which is stored in elasticsearch. It is
+  A document is a JSON document which is stored in Elasticsearch. It is
   like a row in a table in a relational database. Each document is
   stored in an <> and has a <> and an
   <>.
@@ -82,7 +82,7 @@
 
 [[glossary-node]] node ::
 
-  A node is a running instance of elasticsearch which belongs to a
+  A node is a running instance of Elasticsearch which belongs to a
   <>. Multiple nodes can be started on a single
   server for testing purposes, but usually you should have one node per
   server.
@@ -136,7 +136,7 @@
 [[glossary-shard]] shard ::
 
   A shard is a single Lucene instance. It is a low-level “worker” unit
-  which is managed automatically by elasticsearch. An index is a logical
+  which is managed automatically by Elasticsearch. An index is a logical
   namespace which points to <> and
   <> shards.
   +
@@ -159,7 +159,7 @@
 
 [[glossary-term]] term ::
 
-  A term is an exact value that is indexed in elasticsearch. The terms
+  A term is an exact value that is indexed in Elasticsearch. The terms
   `foo`, `Foo`, `FOO` are NOT equivalent. Terms (i.e. exact values) can
   be searched for using _term_ queries. +
    See also <> and <>.
diff --git a/docs/reference/how-to/disk-usage.asciidoc b/docs/reference/how-to/disk-usage.asciidoc
index aa39f28b2fd..62218264bc9 100644
--- a/docs/reference/how-to/disk-usage.asciidoc
+++ b/docs/reference/how-to/disk-usage.asciidoc
@@ -4,7 +4,7 @@
 [float]
 === Disable the features you do not need
 
-By default elasticsearch indexes and adds doc values to most fields so that they
+By default Elasticsearch indexes and adds doc values to most fields so that they
 can be searched and aggregated out of the box. For instance if you have a numeric
 field called `foo` that you need to run histograms on but that you never need to
 filter on, you can safely disable indexing on this field in your
@@ -30,7 +30,7 @@ PUT index
 
 <> fields store normalization factors in the index in order to be
 able to score documents. If you only need matching capabilities on a `text`
-field but do not care about the produced scores, you can configure elasticsearch
+field but do not care about the produced scores, you can configure Elasticsearch
 to not write norms to the index:
 
 [source,js]
@@ -54,7 +54,7 @@ PUT index
 <> fields also store frequencies and positions in the index by
 default. Frequencies are used to compute scores and positions are used to run
 phrase queries. If you do not need to run phrase queries, you can tell
-elasticsearch to not index positions:
+Elasticsearch to not index positions:
 
 [source,js]
 --------------------------------------------------
@@ -75,7 +75,7 @@ PUT index
 // CONSOLE
 
 Furthermore if you do not care about scoring either, you can configure
-elasticsearch to just index matching documents for every term. You will
+Elasticsearch to just index matching documents for every term. You will
 still be able to search on this field, but phrase queries will raise errors
 and scoring will assume that terms appear only once in every document.
 
diff --git a/docs/reference/how-to/indexing-speed.asciidoc b/docs/reference/how-to/indexing-speed.asciidoc
index db7479f9f7d..3661d8ce07d 100644
--- a/docs/reference/how-to/indexing-speed.asciidoc
+++ b/docs/reference/how-to/indexing-speed.asciidoc
@@ -64,13 +64,13 @@ process by <>.
 === Give memory to the filesystem cache
 
 The filesystem cache will be used in order to buffer I/O operations. You should
-make sure to give at least half the memory of the machine running elasticsearch
+make sure to give at least half the memory of the machine running Elasticsearch
 to the filesystem cache.
 
 [float]
 === Use auto-generated ids
 
-When indexing a document that has an explicit id, elasticsearch needs to check
+When indexing a document that has an explicit id, Elasticsearch needs to check
 whether a document with the same id already exists within the same shard, which
 is a costly operation and gets even more costly as the index grows. By using
 auto-generated ids, Elasticsearch can skip this check, which makes indexing
diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc
index 60168ab856d..db84acd3516 100644
--- a/docs/reference/how-to/search-speed.asciidoc
+++ b/docs/reference/how-to/search-speed.asciidoc
@@ -6,7 +6,7 @@
 
 Elasticsearch heavily relies on the filesystem cache in order to make search
 fast. In general, you should make sure that at least half the available memory
-goes to the filesystem cache so that elasticsearch can keep hot regions of the
+goes to the filesystem cache so that Elasticsearch can keep hot regions of the
 index in physical memory.
 
 [float]
@@ -275,8 +275,8 @@ merging to the background merge process.
 Global ordinals are a data-structure that is used in order to run
 <> aggregations on
 <> fields. They are loaded lazily in memory because
-elasticsearch does not know which fields will be used in `terms` aggregations
-and which fields won't. You can tell elasticsearch to load global ordinals
+Elasticsearch does not know which fields will be used in `terms` aggregations
+and which fields won't. You can tell Elasticsearch to load global ordinals
 eagerly at refresh-time by configuring mappings as described below:
 
 [source,js]
@@ -300,7 +300,7 @@ PUT index
 [float]
 === Warm up the filesystem cache
 
-If the machine running elasticsearch is restarted, the filesystem cache will be
+If the machine running Elasticsearch is restarted, the filesystem cache will be
 empty, so it will take some time before the operating system loads hot regions
 of the index into memory so that search operations are fast. You can explicitly
 tell the operating system which files should be loaded into memory eagerly
diff --git a/docs/reference/index-modules/index-sorting.asciidoc b/docs/reference/index-modules/index-sorting.asciidoc
index 8aede7492df..3947b2501c2 100644
--- a/docs/reference/index-modules/index-sorting.asciidoc
+++ b/docs/reference/index-modules/index-sorting.asciidoc
@@ -3,7 +3,7 @@
 
 beta[]
 
-When creating a new index in elasticsearch it is possible to configure how the Segments
+When creating a new index in Elasticsearch it is possible to configure how the Segments
 inside each Shard will be sorted. By default Lucene does not apply any sort.
 The `index.sort.*` settings define which fields should be used to sort the documents inside each Segment.
 
@@ -112,7 +112,7 @@ before activating this feature.
 [[early-terminate]]
 === Early termination of search request
 
-By default in elasticsearch a search request must visit every document that match a query to
+By default in Elasticsearch a search request must visit every document that match a query to
 retrieve the top documents sorted by a specified sort.
 Though when the index sort and the search sort are the same it is possible to limit
 the number of documents that should be visited per segment to retrieve the N top ranked documents globally.
diff --git a/docs/reference/index-modules/merge.asciidoc b/docs/reference/index-modules/merge.asciidoc
index 7e5260f95d4..97db09ba656 100644
--- a/docs/reference/index-modules/merge.asciidoc
+++ b/docs/reference/index-modules/merge.asciidoc
@@ -1,7 +1,7 @@
 [[index-modules-merge]]
 == Merge
 
-A shard in elasticsearch is a Lucene index, and a Lucene index is broken down
+A shard in Elasticsearch is a Lucene index, and a Lucene index is broken down
 into segments. Segments are internal storage elements in the index where the
 index data is stored, and are immutable. Smaller segments are periodically
 merged into larger segments to keep the index size at bay and to expunge
diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc
index 27b5b751233..e657affb607 100644
--- a/docs/reference/index-modules/store.asciidoc
+++ b/docs/reference/index-modules/store.asciidoc
@@ -8,7 +8,7 @@ The store module allows you to control how index data is stored and accessed on
 === File system storage types
 
 There are different file system implementations or _storage types_. By default,
-elasticsearch will pick the best implementation based on the operating
+Elasticsearch will pick the best implementation based on the operating
 environment.
 
 This can be overridden for all indices by adding this to the
@@ -76,7 +76,7 @@ compatibility.
 
 NOTE: This is an expert setting, the details of which may change in the future.
 
-By default, elasticsearch completely relies on the operating system file system
+By default, Elasticsearch completely relies on the operating system file system
 cache for caching I/O operations. It is possible to set `index.store.preload`
 in order to tell the operating system to load the content of hot index
 files into memory upon opening. This setting accept a comma-separated list of
@@ -115,7 +115,7 @@ The default value is the empty array, which means that nothing will be loaded
 into the file-system cache eagerly. For indices that are actively searched,
 you might want to set it to `["nvd", "dvd"]`, which will cause norms and doc
 values to be loaded eagerly into physical memory. These are the two first
-extensions to look at since elasticsearch performs random access on them.
+extensions to look at since Elasticsearch performs random access on them.
 
 A wildcard can be used in order to indicate that all files should be preloaded:
 `index.store.preload: ["*"]`. Note however that it is generally not useful to
diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc
index c8a24ba4614..90d4ae1de56 100644
--- a/docs/reference/mapping/dynamic/templates.asciidoc
+++ b/docs/reference/mapping/dynamic/templates.asciidoc
@@ -257,9 +257,9 @@ Here are some examples of potentially useful dynamic templates:
 
 ===== Structured search
 
-By default elasticsearch will map string fields as a `text` field with a sub
+By default Elasticsearch will map string fields as a `text` field with a sub
 `keyword` field. However if you are only indexing structured content and not
-interested in full text search, you can make elasticsearch map your fields
+interested in full text search, you can make Elasticsearch map your fields
 only as `keyword`s. Note that this means that in order to search those fields,
 you will have to search on the exact same value that was indexed.
 
@@ -290,7 +290,7 @@ PUT my_index
 On the contrary to the previous example, if the only thing that you care about
 on your string fields is full-text search, and if you don't plan on running
 aggregations, sorting or exact search on your string fields, you could tell
-elasticsearch to map it only as a text field (which was the default behaviour
+Elasticsearch to map it only as a text field (which was the default behaviour
 before 5.0):
 
 [source,js]
@@ -357,7 +357,7 @@ remove it as described in the previous section.
 
 ===== Time-series
 
-When doing time series analysis with elasticsearch, it is common to have many
+When doing time series analysis with Elasticsearch, it is common to have many
 numeric fields that you will often aggregate on but never filter on. In such a
 case, you could disable indexing on those fields to save disk space and also
 maybe gain some indexing speed:
diff --git a/docs/reference/modules/discovery/zen.asciidoc b/docs/reference/modules/discovery/zen.asciidoc
index e68350515ea..0cce897f115 100644
--- a/docs/reference/modules/discovery/zen.asciidoc
+++ b/docs/reference/modules/discovery/zen.asciidoc
@@ -1,7 +1,7 @@
 [[modules-discovery-zen]]
 === Zen Discovery
 
-The zen discovery is the built in discovery module for elasticsearch and
+The zen discovery is the built in discovery module for Elasticsearch and
 the default. It provides unicast discovery, but can be extended to
 support cloud environments and other forms of discovery.
 
diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc
index 065c91349c2..a83270ec2aa 100644
--- a/docs/reference/modules/http.asciidoc
+++ b/docs/reference/modules/http.asciidoc
@@ -1,7 +1,7 @@
 [[modules-http]]
 == HTTP
 
-The http module allows to expose *elasticsearch* APIs
+The http module allows to expose *Elasticsearch* APIs
 over HTTP.
 
 The http mechanism is completely asynchronous in nature, meaning that
@@ -74,7 +74,7 @@ allowed. If you prepend and append a `/` to the value, this will
 be treated as a regular expression, allowing you to support HTTP and HTTPs.
 for example using `/https?:\/\/localhost(:[0-9]+)?/` would return the
 request header appropriately in both cases. `*` is a valid value but is
-considered a *security risk* as your elasticsearch instance is open to cross origin
+considered a *security risk* as your Elasticsearch instance is open to cross origin
 requests from *anywhere*.
 
 |`http.cors.max-age` |Browsers send a "preflight" OPTIONS-request to
diff --git a/docs/reference/modules/memcached.asciidoc b/docs/reference/modules/memcached.asciidoc
index b3845c74a3c..508d328671b 100644
--- a/docs/reference/modules/memcached.asciidoc
+++ b/docs/reference/modules/memcached.asciidoc
@@ -1,7 +1,7 @@
 [[modules-memcached]]
 == memcached
 
-The memcached module allows to expose *elasticsearch*
+The memcached module allows to expose *Elasticsearch*
 APIs over the memcached protocol (as closely
 as possible).
 
@@ -18,7 +18,7 @@ automatically detecting the correct one to use.
 === Mapping REST to Memcached Protocol
 
 Memcached commands are mapped to REST and handled by the same generic
-REST layer in elasticsearch. Here is a list of the memcached commands
+REST layer in Elasticsearch. Here is a list of the memcached commands
 supported:
 
 [float]
diff --git a/docs/reference/modules/plugins.asciidoc b/docs/reference/modules/plugins.asciidoc
index ad708e88024..0485065d2b7 100644
--- a/docs/reference/modules/plugins.asciidoc
+++ b/docs/reference/modules/plugins.asciidoc
@@ -4,7 +4,7 @@
 [float]
 === Plugins
 
-Plugins are a way to enhance the basic elasticsearch functionality in a
+Plugins are a way to enhance the basic Elasticsearch functionality in a
 custom manner. They range from adding custom mapping types, custom
 analyzers (in a more built in fashion), custom script engines, custom discovery
 and more.
diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc
index f5b56160049..b8883173b98 100644
--- a/docs/reference/modules/snapshots.asciidoc
+++ b/docs/reference/modules/snapshots.asciidoc
@@ -319,7 +319,7 @@ GET /_snapshot/my_backup/snapshot_1
 // TEST[continued]
 
 This command returns basic information about the snapshot including start and end time, version of
-elasticsearch that created the snapshot, the list of included indices, the current state of the
+Elasticsearch that created the snapshot, the list of included indices, the current state of the
 snapshot and the list of failures that occurred during the snapshot. The snapshot `state` can be
 
 [horizontal]
@@ -343,7 +343,7 @@ snapshot and the list of failures that occurred during the snapshot. The snapsho
 
 `INCOMPATIBLE`::
 
-  The snapshot was created with an old version of elasticsearch and therefore is incompatible with
+  The snapshot was created with an old version of Elasticsearch and therefore is incompatible with
   the current version of the cluster.
 
 
diff --git a/docs/reference/modules/thrift.asciidoc b/docs/reference/modules/thrift.asciidoc
index fc0d08cdd7b..1ea3f818126 100644
--- a/docs/reference/modules/thrift.asciidoc
+++ b/docs/reference/modules/thrift.asciidoc
@@ -2,7 +2,7 @@
 == Thrift
 
 The https://thrift.apache.org/[thrift] transport module allows to expose the REST interface of
-elasticsearch using thrift. Thrift should provide better performance
+Elasticsearch using thrift. Thrift should provide better performance
 over http. Since thrift provides both the wire protocol and the
 transport, it should make using Elasticsearch more efficient (though it has limited
 documentation).
diff --git a/docs/reference/modules/transport.asciidoc b/docs/reference/modules/transport.asciidoc
index 8ed36ce8e74..50c35a4a736 100644
--- a/docs/reference/modules/transport.asciidoc
+++ b/docs/reference/modules/transport.asciidoc
@@ -12,7 +12,7 @@ that there is no blocking thread waiting for a response. The benefit of
 using asynchronous communication is first solving the
 http://en.wikipedia.org/wiki/C10k_problem[C10k problem], as well as
 being the ideal solution for scatter (broadcast) / gather operations such
-as search in ElasticSearch.
+as search in Elasticsearch.
 
 [float]
 === TCP Transport
diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
index c864c643c8f..b2444535153 100644
--- a/docs/reference/search/profile.asciidoc
+++ b/docs/reference/search/profile.asciidoc
@@ -401,7 +401,7 @@ Looking at the previous example:
 We see a single collector named `SimpleTopScoreDocCollector` wrapped into `CancellableCollector`. `SimpleTopScoreDocCollector` is the default "scoring and sorting"
 `Collector` used by Elasticsearch.  The `reason` field attempts to give a plain english description of the class name.  The
 `time_in_nanos` is similar to the time in the Query tree: a wall-clock time inclusive of all children.  Similarly, `children` lists
-all sub-collectors. The `CancellableCollector` that wraps `SimpleTopScoreDocCollector` is used by elasticsearch to detect if the current
+all sub-collectors. The `CancellableCollector` that wraps `SimpleTopScoreDocCollector` is used by Elasticsearch to detect if the current
 search was cancelled and stop collecting documents as soon as it occurs.
 
 It should be noted that Collector times are **independent** from the Query times.  They are calculated, combined
diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc
index 08c6f7fd903..226782d9f57 100644
--- a/docs/reference/search/request/sort.asciidoc
+++ b/docs/reference/search/request/sort.asciidoc
@@ -143,7 +143,7 @@ favor of the options documented above.
 ===== Nested sorting examples
 
 In the below example `offer` is a field of type `nested`.
-The nested `path` needs to be specified; otherwise, elasticsearch doesn't know on what nested level sort values need to be captured.
+The nested `path` needs to be specified; otherwise, Elasticsearch doesn't know on what nested level sort values need to be captured.
 
 [source,js]
 --------------------------------------------------
@@ -171,7 +171,7 @@ POST /_search
 // CONSOLE
 
 In the below example `parent` and `child` fields are of type `nested`.
-The `nested_path` needs to be specified at each level; otherwise, elasticsearch doesn't know on what nested level sort values need to be captured.
+The `nested_path` needs to be specified at each level; otherwise, Elasticsearch doesn't know on what nested level sort values need to be captured.
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc
index cc2b9b7cf8f..625dd944109 100644
--- a/docs/reference/search/search-template.asciidoc
+++ b/docs/reference/search/search-template.asciidoc
@@ -26,7 +26,7 @@ For more information on how Mustache templating and what kind of templating you
 can do with it check out the http://mustache.github.io/mustache.5.html[online
 documentation of the mustache project].
 
-NOTE: The mustache language is implemented in elasticsearch as a sandboxed
+NOTE: The mustache language is implemented in Elasticsearch as a sandboxed
 scripting language, hence it obeys settings that may be used to enable or
 disable scripts per type and context as described in the
 <>
diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc
index 816cf54c529..77ae8a47dd9 100644
--- a/docs/reference/search/search.asciidoc
+++ b/docs/reference/search/search.asciidoc
@@ -60,8 +60,8 @@ GET /_search?q=tag:wow
 // CONSOLE
 // TEST[setup:twitter]
 
-By default elasticsearch doesn't reject any search requests based on the number
-of shards the request hits. While elasticsearch will optimize the search execution
+By default Elasticsearch doesn't reject any search requests based on the number
+of shards the request hits. While Elasticsearch will optimize the search execution
 on the coordinating node a large number of shards can have a significant impact
 CPU and memory wise. It is usually a better idea to organize data in such a way
 that there are fewer larger shards. In case you would like to configure a soft
diff --git a/docs/reference/setup/install/sysconfig-file.asciidoc b/docs/reference/setup/install/sysconfig-file.asciidoc
index 76a9fa22bfc..d2df5cd4e0c 100644
--- a/docs/reference/setup/install/sysconfig-file.asciidoc
+++ b/docs/reference/setup/install/sysconfig-file.asciidoc
@@ -19,7 +19,7 @@
     information, check the
     https://github.com/torvalds/linux/blob/master/Documentation/sysctl/vm.txt[linux kernel documentation]
     about `max_map_count`. This is set via `sysctl` before starting
-    elasticsearch. Defaults to `262144`.
+    Elasticsearch. Defaults to `262144`.
 
 `ES_PATH_CONF`::
 
@@ -34,7 +34,7 @@
 `RESTART_ON_UPGRADE`::
 
     Configure restart on package upgrade, defaults to `false`. This means you
-    will have to restart your elasticsearch instance after installing a
+    will have to restart your Elasticsearch instance after installing a
     package manually. The reason for this is to ensure, that upgrades in a
     cluster do not result in a continuous shard reallocation resulting in high
     network traffic and reducing the response times of your cluster.
diff --git a/docs/reference/setup/secure-settings.asciidoc b/docs/reference/setup/secure-settings.asciidoc
index 2eaebe4ec28..76f7de6b9c1 100644
--- a/docs/reference/setup/secure-settings.asciidoc
+++ b/docs/reference/setup/secure-settings.asciidoc
@@ -2,11 +2,11 @@
 === Secure Settings
 
 Some settings are sensitive, and relying on filesystem permissions to protect
-their values is not sufficient. For this use case, elasticsearch provides a
+their values is not sufficient. For this use case, Elasticsearch provides a
 keystore, which may be password protected, and the `elasticsearch-keystore`
 tool to manage the settings in the keystore.
 
-NOTE: All commands here should be run as the user which will run elasticsearch.
+NOTE: All commands here should be run as the user which will run Elasticsearch.
 
 NOTE: Only some settings are designed to be read from the keystore. See
 documentation for each setting to see if it is supported as part of the keystore.
diff --git a/docs/reference/testing.asciidoc b/docs/reference/testing.asciidoc
index fd1a460f1c5..f4e4b86acc5 100644
--- a/docs/reference/testing.asciidoc
+++ b/docs/reference/testing.asciidoc
@@ -3,7 +3,7 @@
 
 [partintro]
 --
-This section is about utilizing elasticsearch as part of your testing infrastructure.
+This section is about utilizing Elasticsearch as part of your testing infrastructure.
 
 [float]
 [[testing-header]]
diff --git a/docs/reference/testing/testing-framework.asciidoc b/docs/reference/testing/testing-framework.asciidoc
index d1fe769f3c1..f5634a2558e 100644
--- a/docs/reference/testing/testing-framework.asciidoc
+++ b/docs/reference/testing/testing-framework.asciidoc
@@ -3,7 +3,7 @@
 
 [[testing-intro]]
 
-Testing is a crucial part of your application, and as information retrieval itself is already a complex topic, there should not be any additional complexity in setting up a testing infrastructure, which uses elasticsearch. This is the main reason why we decided to release an additional file to the release, which allows you to use the same testing infrastructure we do in the elasticsearch core. The testing framework allows you to setup clusters with multiple nodes in order to check if your code covers everything needed to run in a cluster. The framework prevents you from writing complex code yourself to start, stop or manage several test nodes in a cluster. In addition there is another very important feature called randomized testing, which you are getting for free as it is part of the elasticsearch infrastructure.
+Testing is a crucial part of your application, and as information retrieval itself is already a complex topic, there should not be any additional complexity in setting up a testing infrastructure, which uses Elasticsearch. This is the main reason why we decided to release an additional file to the release, which allows you to use the same testing infrastructure we do in the Elasticsearch core. The testing framework allows you to setup clusters with multiple nodes in order to check if your code covers everything needed to run in a cluster. The framework prevents you from writing complex code yourself to start, stop or manage several test nodes in a cluster. In addition there is another very important feature called randomized testing, which you are getting for free as it is part of the Elasticsearch infrastructure.
 
 
 
@@ -16,9 +16,9 @@ All of the tests are run using a custom junit runner, the `RandomizedRunner` pro
 
 
 [[using-elasticsearch-test-classes]]
-=== Using the elasticsearch test classes
+=== Using the Elasticsearch test classes
 
-First, you need to include the testing dependency in your project, along with the elasticsearch dependency you have already added. If you use maven and its `pom.xml` file, it looks like this
+First, you need to include the testing dependency in your project, along with the Elasticsearch dependency you have already added. If you use maven and its `pom.xml` file, it looks like this
 
 [source,xml]
 --------------------------------------------------
@@ -50,7 +50,7 @@ We provide a few classes that you can inherit from in your own test classes whic
 [[unit-tests]]
 === unit tests
 
-If your test is a well isolated unit test which doesn't need a running elasticsearch cluster, you can use the `ESTestCase`. If you are testing lucene features, use `ESTestCase` and if you are testing concrete token streams, use the `ESTokenStreamTestCase` class. Those specific classes execute additional checks which ensure that no resources leaks are happening, after the test has run.
+If your test is a well isolated unit test which doesn't need a running Elasticsearch cluster, you can use the `ESTestCase`. If you are testing lucene features, use `ESTestCase` and if you are testing concrete token streams, use the `ESTokenStreamTestCase` class. Those specific classes execute additional checks which ensure that no resources leaks are happening, after the test has run.
 
 
 [[integration-tests]]
@@ -58,7 +58,7 @@ If your test is a well isolated unit test which doesn't need a running elasticse
 
 These kind of tests require firing up a whole cluster of nodes, before the tests can actually be run. Compared to unit tests they are obviously way more time consuming, but the test infrastructure tries to minimize the time cost by only restarting the whole cluster, if this is configured explicitly.
 
-The class your tests have to inherit from is `ESIntegTestCase`. By inheriting from this class, you will no longer need to start elasticsearch nodes manually in your test, although you might need to ensure that at least a certain number of nodes are up. The integration test behaviour can be configured heavily by specifying different system properties on test runs. See the `TESTING.asciidoc` documentation in the https://github.com/elastic/elasticsearch/blob/master/TESTING.asciidoc[source repository] for more information.
+The class your tests have to inherit from is `ESIntegTestCase`. By inheriting from this class, you will no longer need to start Elasticsearch nodes manually in your test, although you might need to ensure that at least a certain number of nodes are up. The integration test behaviour can be configured heavily by specifying different system properties on test runs. See the `TESTING.asciidoc` documentation in the https://github.com/elastic/elasticsearch/blob/master/TESTING.asciidoc[source repository] for more information.
 
 
 [[number-of-shards]]
@@ -100,8 +100,8 @@ The `InternalTestCluster` class is the heart of the cluster functionality in a r
 `stopRandomNode()`::                Stop a random node in your cluster to mimic an outage
 `stopCurrentMasterNode()`::         Stop the current master node to force a new election
 `stopRandomNonMaster()`::           Stop a random non master node to mimic an outage
-`buildNode()`::                     Create a new elasticsearch node
-`startNode(settings)`::             Create and start a new elasticsearch node
+`buildNode()`::                     Create a new Elasticsearch node
+`startNode(settings)`::             Create and start a new Elasticsearch node
 
 
 [[changing-node-settings]]
@@ -165,7 +165,7 @@ data nodes will be allowed to become masters as well.
 [[changing-node-configuration]]
 ==== Changing plugins via configuration
 
-As elasticsearch is using JUnit 4, using the `@Before` and `@After` annotations is not a problem. However you should keep in mind, that this does not have any effect in your cluster setup, as the cluster is already up and running when those methods are run. So in case you want to configure settings - like loading a plugin on node startup - before the node is actually running, you should overwrite the `nodePlugins()` method from the `ESIntegTestCase` class and return the plugin classes each node should load.
+As Elasticsearch is using JUnit 4, using the `@Before` and `@After` annotations is not a problem. However you should keep in mind, that this does not have any effect in your cluster setup, as the cluster is already up and running when those methods are run. So in case you want to configure settings - like loading a plugin on node startup - before the node is actually running, you should overwrite the `nodePlugins()` method from the `ESIntegTestCase` class and return the plugin classes each node should load.
 
 [source,java]
 -----------------------------------------
@@ -191,7 +191,7 @@ The next step is to convert your test using static test data into a test using r
 * Changing your response sizes/configurable limits with each run
 * Changing the number of shards/replicas when creating an index
 
-So, how can you create random data. The most important thing to know is, that you never should instantiate your own `Random` instance, but use the one provided in the `RandomizedTest`, from which all elasticsearch dependent test classes inherit from.
+So, how can you create random data. The most important thing to know is, that you never should instantiate your own `Random` instance, but use the one provided in the `RandomizedTest`, from which all Elasticsearch dependent test classes inherit from.
 
 [horizontal]
 `getRandom()`::         Returns the random instance, which can recreated when calling the test with specific parameters
@@ -221,7 +221,7 @@ If you want to debug a specific problem with a specific random seed, you can use
 [[assertions]]
 === Assertions
 
-As many elasticsearch tests are checking for a similar output, like the amount of hits or the first hit or special highlighting, a couple of predefined assertions have been created. Those have been put into the `ElasticsearchAssertions` class. There is also a specific geo assertions in `ElasticsearchGeoAssertions`.
+As many Elasticsearch tests are checking for a similar output, like the amount of hits or the first hit or special highlighting, a couple of predefined assertions have been created. Those have been put into the `ElasticsearchAssertions` class. There is also a specific geo assertions in `ElasticsearchGeoAssertions`.
 
 [horizontal]
 `assertHitCount()`::        Checks hit count of a search or count request

From 65e602c2be3529db0a764ebb6f066403876e3485 Mon Sep 17 00:00:00 2001
From: Clinton Gormley 
Date: Wed, 29 Nov 2017 13:04:38 +0100
Subject: [PATCH 120/297] Update index-modules.asciidoc

Docs: Clarified `blocks.write` vs `blocks.read_only`
---
 docs/reference/index-modules.asciidoc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index 6dd0fad4365..999bd948a15 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -180,7 +180,9 @@ specific index module:
 
 `index.blocks.write`::
 
-    Set to `true` to disable write operations against the index.
+    Set to `true` to disable data write operations against the index. Unlike `read_only', 
+    this setting does not affect metadata. For instance, you can close an index with a `write`
+    block, but not an index with a `read_only` block.
 
 `index.blocks.metadata`::
 

From 7bfb2737639e9182cfee29c76e40ac322ceaf449 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?=
 <10398885+cbuescher@users.noreply.github.com>
Date: Wed, 29 Nov 2017 15:19:16 +0100
Subject: [PATCH 121/297] Add k parameter to PrecisionAtK metric (#27569)

---
 .../index/rankeval/EvaluationMetric.java      |  9 +++++
 .../index/rankeval/PrecisionAtK.java          | 32 ++++++++++++---
 .../rankeval/TransportRankEvalAction.java     |  7 +++-
 .../index/rankeval/PrecisionAtKTests.java     | 39 ++++++++++++------
 .../index/rankeval/RankEvalRequestIT.java     | 40 +++++++++++++------
 5 files changed, 96 insertions(+), 31 deletions(-)

diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
index 444666ee0d4..982d8fa8d30 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java
@@ -89,4 +89,13 @@ public interface EvaluationMetric extends ToXContent, NamedWriteable {
     default double combine(Collection partialResults) {
         return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size();
     }
+
+    /**
+     * Metrics can define a size of the search hits windows they want to retrieve by overwriting
+     * this method. The default implementation returns an empty optional.
+     * @return the number of search hits this metrics requests
+     */
+    default Optional forcedSearchSize() {
+        return Optional.empty();
+    }
 }
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java
index 9152141043c..4beeeea2b40 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java
@@ -45,6 +45,7 @@ import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRati
  * relevant_rating_threshold` parameter.

  * The `ignore_unlabeled` parameter (default to false) controls if unrated
  * documents should be ignored.
+ * The `k` parameter (defaults to 10) controls the search window size.
  */
 public class PrecisionAtK implements EvaluationMetric {
 
@@ -52,9 +53,13 @@ public class PrecisionAtK implements EvaluationMetric {
 
     private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold");
     private static final ParseField IGNORE_UNLABELED_FIELD = new ParseField("ignore_unlabeled");
+    private static final ParseField K_FIELD = new ParseField("k");
+
+    private static final int DEFAULT_K = 10;
 
     private final boolean ignoreUnlabeled;
     private final int relevantRatingThreshhold;
+    private final int k;
 
     /**
      * Metric implementing Precision@K.
@@ -65,42 +70,52 @@ public class PrecisionAtK implements EvaluationMetric {
      *            Set to 'true', unlabeled documents are ignored and neither count
      *            as true or false positives. Set to 'false', they are treated as
      *            false positives.
+     * @param k
+     *            controls the window size for the search results the metric takes into account
      */
-    public PrecisionAtK(int threshold, boolean ignoreUnlabeled) {
+    public PrecisionAtK(int threshold, boolean ignoreUnlabeled, int k) {
         if (threshold < 0) {
             throw new IllegalArgumentException("Relevant rating threshold for precision must be positive integer.");
         }
+        if (k <= 0) {
+            throw new IllegalArgumentException("Window size k must be positive.");
+        }
         this.relevantRatingThreshhold = threshold;
         this.ignoreUnlabeled = ignoreUnlabeled;
+        this.k = k;
     }
 
     public PrecisionAtK() {
-        this(1, false);
+        this(1, false, DEFAULT_K);
     }
 
-
     private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME,
             args -> {
                 Integer threshHold = (Integer) args[0];
                 Boolean ignoreUnlabeled = (Boolean) args[1];
+                Integer k = (Integer) args[2];
                 return new PrecisionAtK(threshHold == null ? 1 : threshHold,
-                        ignoreUnlabeled == null ? false : ignoreUnlabeled);
+                        ignoreUnlabeled == null ? false : ignoreUnlabeled,
+                                k == null ? 10 : k);
             });
 
     static {
         PARSER.declareInt(optionalConstructorArg(), RELEVANT_RATING_FIELD);
         PARSER.declareBoolean(optionalConstructorArg(), IGNORE_UNLABELED_FIELD);
+        PARSER.declareInt(optionalConstructorArg(), K_FIELD);
     }
 
     PrecisionAtK(StreamInput in) throws IOException {
         relevantRatingThreshhold = in.readVInt();
         ignoreUnlabeled = in.readBoolean();
+        k = in.readVInt();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(relevantRatingThreshhold);
         out.writeBoolean(ignoreUnlabeled);
+        out.writeVInt(k);
     }
 
     @Override
@@ -123,6 +138,11 @@ public class PrecisionAtK implements EvaluationMetric {
         return ignoreUnlabeled;
     }
 
+    @Override
+    public Optional forcedSearchSize() {
+        return Optional.of(k);
+    }
+
     public static PrecisionAtK fromXContent(XContentParser parser) {
         return PARSER.apply(parser, null);
     }
@@ -167,6 +187,7 @@ public class PrecisionAtK implements EvaluationMetric {
         builder.startObject(NAME);
         builder.field(RELEVANT_RATING_FIELD.getPreferredName(), this.relevantRatingThreshhold);
         builder.field(IGNORE_UNLABELED_FIELD.getPreferredName(), this.ignoreUnlabeled);
+        builder.field(K_FIELD.getPreferredName(), this.k);
         builder.endObject();
         builder.endObject();
         return builder;
@@ -182,12 +203,13 @@ public class PrecisionAtK implements EvaluationMetric {
         }
         PrecisionAtK other = (PrecisionAtK) obj;
         return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold)
+                && Objects.equals(k, other.k)
                 && Objects.equals(ignoreUnlabeled, other.ignoreUnlabeled);
     }
 
     @Override
     public final int hashCode() {
-        return Objects.hash(relevantRatingThreshhold, ignoreUnlabeled);
+        return Objects.hash(relevantRatingThreshhold, ignoreUnlabeled, k);
     }
 
     static class Breakdown implements MetricDetails {
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
index 0ff491cb8bc..7299869bcf8 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
@@ -85,6 +85,7 @@ public class TransportRankEvalAction extends HandledTransportAction listener) {
         RankEvalSpec evaluationSpecification = request.getRankEvalSpec();
         List indices = evaluationSpecification.getIndices();
+        EvaluationMetric metric = evaluationSpecification.getMetric();
 
         List ratedRequests = evaluationSpecification.getRatedRequests();
         Map errors = new ConcurrentHashMap<>(ratedRequests.size());
@@ -113,6 +114,10 @@ public class TransportRankEvalAction extends HandledTransportAction summaryFields = ratedRequest.getSummaryFields();
             if (summaryFields.isEmpty()) {
@@ -123,7 +128,7 @@ public class TransportRankEvalAction extends HandledTransportAction new PrecisionAtK(-1, false));
+        expectThrows(IllegalArgumentException.class, () -> new PrecisionAtK(-1, false, 10));
+    }
+
+    public void testInvalidK() {
+        expectThrows(IllegalArgumentException.class, () -> new PrecisionAtK(1, false, -10));
     }
 
     public static PrecisionAtK createTestItem() {
-        return new PrecisionAtK(randomIntBetween(0, 10), randomBoolean());
+        return new PrecisionAtK(randomIntBetween(0, 10), randomBoolean(), randomIntBetween(1, 50));
     }
 
     public void testXContentRoundtrip() throws IOException {
@@ -193,16 +196,28 @@ public class PrecisionAtKTests extends ESTestCase {
     }
 
     private static PrecisionAtK copy(PrecisionAtK original) {
-        return new PrecisionAtK(original.getRelevantRatingThreshold(), original.getIgnoreUnlabeled());
+        return new PrecisionAtK(original.getRelevantRatingThreshold(), original.getIgnoreUnlabeled(), original.forcedSearchSize().get());
     }
 
     private static PrecisionAtK mutate(PrecisionAtK original) {
-        if (randomBoolean()) {
-            return new PrecisionAtK(original.getRelevantRatingThreshold(), !original.getIgnoreUnlabeled());
-        } else {
-            return new PrecisionAtK(randomValueOtherThan(original.getRelevantRatingThreshold(), () -> randomIntBetween(0, 10)),
-                    original.getIgnoreUnlabeled());
+        PrecisionAtK pAtK;
+        switch (randomIntBetween(0, 2)) {
+        case 0:
+            pAtK = new PrecisionAtK(original.getRelevantRatingThreshold(), !original.getIgnoreUnlabeled(),
+                    original.forcedSearchSize().get());
+            break;
+        case 1:
+            pAtK = new PrecisionAtK(randomValueOtherThan(original.getRelevantRatingThreshold(), () -> randomIntBetween(0, 10)),
+                    original.getIgnoreUnlabeled(), original.forcedSearchSize().get());
+            break;
+        case 2:
+            pAtK = new PrecisionAtK(original.getRelevantRatingThreshold(),
+                    original.getIgnoreUnlabeled(), original.forcedSearchSize().get() + 1);
+            break;
+        default:
+            throw new IllegalStateException("The test should only allow three parameters mutated");
         }
+        return pAtK;
     }
 
     private static SearchHit[] toSearchHits(List rated, String index) {
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
index 707a1448c81..2ab3e2d9a57 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
@@ -20,10 +20,12 @@
 package org.elasticsearch.index.rankeval;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.Before;
 
@@ -68,6 +70,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         List specifications = new ArrayList<>();
         SearchSourceBuilder testQuery = new SearchSourceBuilder();
         testQuery.query(new MatchAllQueryBuilder());
+        testQuery.sort(FieldSortBuilder.DOC_FIELD_NAME);
         RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query",
                 createRelevant("2", "3", "4", "5"), testQuery);
         amsterdamRequest.addSummaryFields(Arrays.asList(new String[] { "text", "title" }));
@@ -79,7 +82,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
 
         specifications.add(berlinRequest);
 
-        PrecisionAtK metric = new PrecisionAtK(1, true);
+        PrecisionAtK metric = new PrecisionAtK(1, false, 10);
         RankEvalSpec task = new RankEvalSpec(specifications, metric);
         task.addIndices(indices);
 
@@ -89,7 +92,8 @@ public class RankEvalRequestIT extends ESIntegTestCase {
 
         RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request())
                 .actionGet();
-        assertEquals(1.0, response.getEvaluationResult(), Double.MIN_VALUE);
+        double expectedPrecision = (1.0 / 6.0 + 4.0 / 6.0) / 2.0;
+        assertEquals(expectedPrecision, response.getEvaluationResult(), Double.MIN_VALUE);
         Set> entrySet = response.getPartialResults().entrySet();
         assertEquals(2, entrySet.size());
         for (Entry entry : entrySet) {
@@ -121,6 +125,18 @@ public class RankEvalRequestIT extends ESIntegTestCase {
                 }
             }
         }
+
+        // test that a different window size k affects the result
+        metric = new PrecisionAtK(1, false, 3);
+        task = new RankEvalSpec(specifications, metric);
+        task.addIndices(indices);
+
+        builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
+        builder.setRankEvalSpec(task);
+
+        response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
+        expectedPrecision = (1.0 / 3.0 + 2.0 / 3.0) / 2.0;
+        assertEquals(0.5, response.getEvaluationResult(), Double.MIN_VALUE);
     }
 
     /**
@@ -146,18 +162,16 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK());
         task.addIndices(indices);
 
-        RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(),
-                RankEvalAction.INSTANCE, new RankEvalRequest());
-        builder.setRankEvalSpec(task);
-
-        RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request())
-                .actionGet();
-        assertEquals(1, response.getFailures().size());
-        ElasticsearchException[] rootCauses = ElasticsearchException
-                .guessRootCauses(response.getFailures().get("broken_query"));
-        assertEquals("java.lang.NumberFormatException: For input string: \"noStringOnNumericFields\"",
-                rootCauses[0].getCause().toString());
+        try (Client client = client()) {
+            RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client, RankEvalAction.INSTANCE, new RankEvalRequest());
+            builder.setRankEvalSpec(task);
 
+            RankEvalResponse response = client.execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
+            assertEquals(1, response.getFailures().size());
+            ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(response.getFailures().get("broken_query"));
+            assertEquals("java.lang.NumberFormatException: For input string: \"noStringOnNumericFields\"",
+                    rootCauses[0].getCause().toString());
+        }
     }
 
     private static List createRelevant(String... docs) {

From dbf17152d18bd10b456ea353cee0fc66001dc231 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen 
Date: Wed, 29 Nov 2017 17:31:39 +0100
Subject: [PATCH 122/297] docs: use `doc_value_fields` fields as alternative
 for nested inner hits _source fetching

instead of stored fields as doc values are more likely to be enabled by default
---
 .../search/request/inner-hits.asciidoc         | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc
index fceb37418eb..ca459fe801e 100644
--- a/docs/reference/search/request/inner-hits.asciidoc
+++ b/docs/reference/search/request/inner-hits.asciidoc
@@ -200,7 +200,7 @@ its `_source` field. To include the source of just the nested document, the sour
 the relevant bit for the nested document is included as source in the inner hit. Doing this for each matching nested document
 has an impact on the time it takes to execute the entire search request, especially when `size` and the inner hits' `size`
 are set higher than the default. To avoid the relatively expensive source extraction for nested inner hits, one can disable
-including the source and solely rely on stored fields. Like this:
+including the source and solely rely on doc values fields. Like this:
 
 [source,js]
 --------------------------------------------------
@@ -210,13 +210,7 @@ PUT test
     "doc": {
       "properties": {
         "comments": {
-          "type": "nested",
-          "properties": {
-            "text": {
-              "type": "text",
-              "store": true
-            }
-          }
+          "type": "nested"
         }
       }
     }
@@ -248,7 +242,7 @@ POST test/_search
       },
       "inner_hits": {
         "_source" : false,
-        "stored_fields" : ["comments.text"]
+        "docvalue_fields" : ["comments.text.keyword"]
       }
     }
   }
@@ -290,7 +284,7 @@ Response not included in text but tested for completeness sake.
                   },
                   "_score": 1.0444683,
                   "fields": {
-                    "comments.text": [
+                    "comments.text.keyword": [
                       "words words words"
                     ]
                   }
@@ -326,10 +320,6 @@ PUT test
         "comments": {
           "type": "nested",
           "properties": {
-            "message": {
-              "type": "text",
-              "store": true
-            },
             "votes": {
               "type": "nested"
             }

From 29c554032356c658ecf0d9d23068a1c84e9364bb Mon Sep 17 00:00:00 2001
From: David Turner 
Date: Wed, 29 Nov 2017 18:12:18 +0000
Subject: [PATCH 123/297] Remove AwaitsFix

---
 .../action/admin/indices/shards/IndicesShardStoreRequestIT.java  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
index cb150e012ae..b43996ddf02 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
@@ -151,7 +151,6 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
         assertThat(shardStatuses.get(index1).size(), equalTo(2));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/12416")
     public void testCorruptedShards() throws Exception {
         String index = "test";
         internalCluster().ensureAtLeastNumDataNodes(2);

From 55cb8ddd80d9caf401d0eb5030fccc0791b5d822 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Wed, 29 Nov 2017 17:35:00 -0500
Subject: [PATCH 124/297] Do not set data paths on no local storage required

Today when configuring the data paths for the environment, we set data
paths to either the specified path.data or default to data relative to
the Elasticsearch home. Yet if node.local_storage is false, data paths
do not even make sense. In this case, we should reject if path.data is
set, and instead of defaulting data paths to data relative to home, we
should set this to empty paths. This commit does this.

Relates #27587
---
 .../org/elasticsearch/env/Environment.java    | 35 +++++++++++++------
 .../elasticsearch/env/EnvironmentTests.java   | 31 ++++++++++++++++
 .../env/NodeEnvironmentTests.java             |  4 +--
 3 files changed, 57 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java
index 721cdcf9ba6..2433ccf6e8e 100644
--- a/core/src/main/java/org/elasticsearch/env/Environment.java
+++ b/core/src/main/java/org/elasticsearch/env/Environment.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.env;
 
 import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Setting;
@@ -45,6 +46,9 @@ import java.util.function.Function;
 // TODO: move PathUtils to be package-private here instead of
 // public+forbidden api!
 public class Environment {
+
+    private static final Path[] EMPTY_PATH_ARRAY = new Path[0];
+
     public static final Setting PATH_HOME_SETTING = Setting.simpleString("path.home", Property.NodeScope);
     public static final Setting> PATH_DATA_SETTING =
             Setting.listSetting("path.data", Collections.emptyList(), Function.identity(), Property.NodeScope);
@@ -103,16 +107,25 @@ public class Environment {
 
         List dataPaths = PATH_DATA_SETTING.get(settings);
         final ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
-        if (dataPaths.isEmpty() == false) {
-            dataFiles = new Path[dataPaths.size()];
-            dataWithClusterFiles = new Path[dataPaths.size()];
-            for (int i = 0; i < dataPaths.size(); i++) {
-                dataFiles[i] = PathUtils.get(dataPaths.get(i));
-                dataWithClusterFiles[i] = dataFiles[i].resolve(clusterName.value());
+        if (DiscoveryNode.nodeRequiresLocalStorage(settings)) {
+            if (dataPaths.isEmpty() == false) {
+                dataFiles = new Path[dataPaths.size()];
+                dataWithClusterFiles = new Path[dataPaths.size()];
+                for (int i = 0; i < dataPaths.size(); i++) {
+                    dataFiles[i] = PathUtils.get(dataPaths.get(i));
+                    dataWithClusterFiles[i] = dataFiles[i].resolve(clusterName.value());
+                }
+            } else {
+                dataFiles = new Path[]{homeFile.resolve("data")};
+                dataWithClusterFiles = new Path[]{homeFile.resolve("data").resolve(clusterName.value())};
             }
         } else {
-            dataFiles = new Path[]{homeFile.resolve("data")};
-            dataWithClusterFiles = new Path[]{homeFile.resolve("data").resolve(clusterName.value())};
+            if (dataPaths.isEmpty()) {
+                dataFiles = dataWithClusterFiles = EMPTY_PATH_ARRAY;
+            } else {
+                final String paths = String.join(",", dataPaths);
+                throw new IllegalStateException("node does not require local storage yet path.data is set to [" + paths + "]");
+            }
         }
         if (PATH_SHARED_DATA_SETTING.exists(settings)) {
             sharedDataFile = PathUtils.get(PATH_SHARED_DATA_SETTING.get(settings)).normalize();
@@ -120,13 +133,13 @@ public class Environment {
             sharedDataFile = null;
         }
         List repoPaths = PATH_REPO_SETTING.get(settings);
-        if (repoPaths.isEmpty() == false) {
+        if (repoPaths.isEmpty()) {
+            repoFiles = EMPTY_PATH_ARRAY;
+        } else {
             repoFiles = new Path[repoPaths.size()];
             for (int i = 0; i < repoPaths.size(); i++) {
                 repoFiles[i] = PathUtils.get(repoPaths.get(i));
             }
-        } else {
-            repoFiles = new Path[0];
         }
 
         // this is trappy, Setting#get(Settings) will get a fallback setting yet return false for Settings#exists(Settings)
diff --git a/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java
index 6ddf6b3ba73..5ca3f4dc6a5 100644
--- a/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java
+++ b/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java
@@ -28,7 +28,10 @@ import java.nio.file.Path;
 import static org.hamcrest.CoreMatchers.endsWith;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.Matchers.arrayWithSize;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasToString;
 
 /**
  * Simple unit-tests for Environment.java
@@ -115,4 +118,32 @@ public class EnvironmentTests extends ESTestCase {
         assertThat(environment.configFile(), equalTo(pathHome.resolve("config")));
     }
 
+    public void testNodeDoesNotRequireLocalStorage() {
+        final Path pathHome = createTempDir().toAbsolutePath();
+        final Settings settings =
+                Settings.builder()
+                        .put("path.home", pathHome)
+                        .put("node.local_storage", false)
+                        .put("node.master", false)
+                        .put("node.data", false)
+                        .build();
+        final Environment environment = new Environment(settings, null);
+        assertThat(environment.dataFiles(), arrayWithSize(0));
+    }
+
+    public void testNodeDoesNotRequireLocalStorageButHasPathData() {
+        final Path pathHome = createTempDir().toAbsolutePath();
+        final Path pathData = pathHome.resolve("data");
+        final Settings settings =
+                Settings.builder()
+                        .put("path.home", pathHome)
+                        .put("path.data", pathData)
+                        .put("node.local_storage", false)
+                        .put("node.master", false)
+                        .put("node.data", false)
+                        .build();
+        final IllegalStateException e = expectThrows(IllegalStateException.class, () -> new Environment(settings, null));
+        assertThat(e, hasToString(containsString("node does not require local storage yet path.data is set to [" + pathData + "]")));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
index 90161e5faaf..40193ebc363 100644
--- a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
+++ b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
@@ -397,14 +397,14 @@ public class NodeEnvironmentTests extends ESTestCase {
     }
 
     public void testPersistentNodeId() throws IOException {
-        String[] paths = tmpPaths();
-        NodeEnvironment env = newNodeEnvironment(paths, Settings.builder()
+        NodeEnvironment env = newNodeEnvironment(new String[0], Settings.builder()
             .put("node.local_storage", false)
             .put("node.master", false)
             .put("node.data", false)
             .build());
         String nodeID = env.nodeId();
         env.close();
+        final String[] paths = tmpPaths();
         env = newNodeEnvironment(paths, Settings.EMPTY);
         assertThat("previous node didn't have local storage enabled, id should change", env.nodeId(), not(equalTo(nodeID)));
         nodeID = env.nodeId();

From b8557651aabd0dca94a5c524ccd3124db5e16fda Mon Sep 17 00:00:00 2001
From: Tim Brooks 
Date: Wed, 29 Nov 2017 15:47:12 -0700
Subject: [PATCH 125/297] Add exception handling for write listeners (#27590)

This potential issue was exposed when I saw this PR #27542. Essentially
we currently execute the write listeners all over the place without
consistently catching and handling exceptions. Some of these exceptions
will be logged in different ways (including as low as `debug`).

This commit adds a single location where these listeners are executed.
If the listener throws an execption, the exception is caught and logged
at the `warn` level.
---
 .../transport/nio/SocketEventHandler.java     | 11 +++++
 .../transport/nio/SocketSelector.java         | 41 +++++++++++++++++--
 .../nio/channel/TcpWriteContext.java          |  6 +--
 .../transport/nio/SocketSelectorTests.java    | 19 +++++++++
 .../nio/channel/TcpWriteContextTests.java     | 10 ++---
 5 files changed, 75 insertions(+), 12 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/SocketEventHandler.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/SocketEventHandler.java
index 50362c5a665..799e81dd4a4 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/SocketEventHandler.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/SocketEventHandler.java
@@ -21,6 +21,7 @@ package org.elasticsearch.transport.nio;
 
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.transport.nio.channel.NioChannel;
 import org.elasticsearch.transport.nio.channel.NioSocketChannel;
 import org.elasticsearch.transport.nio.channel.SelectionKeyUtils;
@@ -146,6 +147,16 @@ public class SocketEventHandler extends EventHandler {
         exceptionCaught((NioSocketChannel) channel, exception);
     }
 
+    /**
+     * This method is called when a listener attached to a channel operation throws an exception.
+     *
+     * @param listener that was called
+     * @param exception that occurred
+     */
+     void listenerException(ActionListener listener, Exception exception) {
+        logger.warn(new ParameterizedMessage("exception while executing listener: {}", listener), exception);
+    }
+
     private void exceptionCaught(NioSocketChannel channel, Exception e) {
         channel.getExceptionContext().accept(channel, e);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/SocketSelector.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/SocketSelector.java
index 5a298b34bb9..12bc51914d6 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/SocketSelector.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/SocketSelector.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.transport.nio;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.transport.nio.channel.NioSocketChannel;
 import org.elasticsearch.transport.nio.channel.SelectionKeyUtils;
 import org.elasticsearch.transport.nio.channel.WriteContext;
@@ -79,7 +80,7 @@ public class SocketSelector extends ESSelector {
     void cleanup() {
         WriteOperation op;
         while ((op = queuedWrites.poll()) != null) {
-            op.getListener().onFailure(new ClosedSelectorException());
+            executeFailedListener(op.getListener(), new ClosedSelectorException());
         }
         channelsToClose.addAll(newChannels);
     }
@@ -107,7 +108,7 @@ public class SocketSelector extends ESSelector {
         if (isOpen() == false) {
             boolean wasRemoved = queuedWrites.remove(writeOperation);
             if (wasRemoved) {
-                writeOperation.getListener().onFailure(new ClosedSelectorException());
+                executeFailedListener(writeOperation.getListener(), new ClosedSelectorException());
             }
         } else {
             wakeup();
@@ -128,7 +129,39 @@ public class SocketSelector extends ESSelector {
             SelectionKeyUtils.setWriteInterested(channel);
             context.queueWriteOperations(writeOperation);
         } catch (Exception e) {
-            writeOperation.getListener().onFailure(e);
+            executeFailedListener(writeOperation.getListener(), e);
+        }
+    }
+
+    /**
+     * Executes a success listener with consistent exception handling. This can only be called from current
+     * selector thread.
+     *
+     * @param listener to be executed
+     * @param value to provide to listener
+     */
+    public  void executeListener(ActionListener listener, V value) {
+        assert isOnCurrentThread() : "Must be on selector thread";
+        try {
+            listener.onResponse(value);
+        } catch (Exception e) {
+            eventHandler.listenerException(listener, e);
+        }
+    }
+
+    /**
+     * Executes a failed listener with consistent exception handling. This can only be called from current
+     * selector thread.
+     *
+     * @param listener to be executed
+     * @param exception to provide to listener
+     */
+    public  void executeFailedListener(ActionListener listener, Exception exception) {
+        assert isOnCurrentThread() : "Must be on selector thread";
+        try {
+            listener.onFailure(exception);
+        } catch (Exception e) {
+            eventHandler.listenerException(listener, e);
         }
     }
 
@@ -154,7 +187,7 @@ public class SocketSelector extends ESSelector {
             if (writeOperation.getChannel().isWritable()) {
                 queueWriteInChannelBuffer(writeOperation);
             } else {
-                writeOperation.getListener().onFailure(new ClosedChannelException());
+                executeFailedListener(writeOperation.getListener(), new ClosedChannelException());
             }
         }
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpWriteContext.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpWriteContext.java
index d38cd1320d1..63b876c0987 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpWriteContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpWriteContext.java
@@ -82,7 +82,7 @@ public class TcpWriteContext implements WriteContext {
     public void clearQueuedWriteOps(Exception e) {
         assert channel.getSelector().isOnCurrentThread() : "Must be on selector thread to clear queued writes";
         for (WriteOperation op : queued) {
-            op.getListener().onFailure(e);
+            channel.getSelector().executeFailedListener(op.getListener(), e);
         }
         queued.clear();
     }
@@ -91,12 +91,12 @@ public class TcpWriteContext implements WriteContext {
         try {
             headOp.flush();
         } catch (IOException e) {
-            headOp.getListener().onFailure(e);
+            channel.getSelector().executeFailedListener(headOp.getListener(), e);
             throw e;
         }
 
         if (headOp.isFullyFlushed()) {
-            headOp.getListener().onResponse(null);
+            channel.getSelector().executeListener(headOp.getListener(), null);
         } else {
             queued.push(headOp);
         }
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketSelectorTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketSelectorTests.java
index 61a9499f8db..34a44ee4e4b 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketSelectorTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketSelectorTests.java
@@ -308,4 +308,23 @@ public class SocketSelectorTests extends ESTestCase {
         verify(eventHandler).handleClose(channel);
         verify(eventHandler).handleClose(unRegisteredChannel);
     }
+
+    public void testExecuteListenerWillHandleException() throws Exception {
+        RuntimeException exception = new RuntimeException();
+        doThrow(exception).when(listener).onResponse(null);
+
+        socketSelector.executeListener(listener, null);
+
+        verify(eventHandler).listenerException(listener, exception);
+    }
+
+    public void testExecuteFailedListenerWillHandleException() throws Exception {
+        IOException ioException = new IOException();
+        RuntimeException exception = new RuntimeException();
+        doThrow(exception).when(listener).onFailure(ioException);
+
+        socketSelector.executeFailedListener(listener, ioException);
+
+        verify(eventHandler).listenerException(listener, exception);
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpWriteContextTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpWriteContextTests.java
index 7e6410b6c61..16b53cd71b0 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpWriteContextTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpWriteContextTests.java
@@ -118,7 +118,7 @@ public class TcpWriteContextTests extends ESTestCase {
         ClosedChannelException e = new ClosedChannelException();
         writeContext.clearQueuedWriteOps(e);
 
-        verify(listener).onFailure(e);
+        verify(selector).executeFailedListener(listener, e);
 
         assertFalse(writeContext.hasQueuedWriteOps());
     }
@@ -136,7 +136,7 @@ public class TcpWriteContextTests extends ESTestCase {
         writeContext.flushChannel();
 
         verify(writeOperation).flush();
-        verify(listener).onResponse(null);
+        verify(selector).executeListener(listener, null);
         assertFalse(writeContext.hasQueuedWriteOps());
     }
 
@@ -173,7 +173,7 @@ public class TcpWriteContextTests extends ESTestCase {
         when(writeOperation2.isFullyFlushed()).thenReturn(false);
         writeContext.flushChannel();
 
-        verify(listener).onResponse(null);
+        verify(selector).executeListener(listener, null);
         verify(listener2, times(0)).onResponse(channel);
         assertTrue(writeContext.hasQueuedWriteOps());
 
@@ -181,7 +181,7 @@ public class TcpWriteContextTests extends ESTestCase {
 
         writeContext.flushChannel();
 
-        verify(listener2).onResponse(null);
+        verify(selector).executeListener(listener2, null);
         assertFalse(writeContext.hasQueuedWriteOps());
     }
 
@@ -198,7 +198,7 @@ public class TcpWriteContextTests extends ESTestCase {
         when(writeOperation.getListener()).thenReturn(listener);
         expectThrows(IOException.class, () -> writeContext.flushChannel());
 
-        verify(listener).onFailure(exception);
+        verify(selector).executeFailedListener(listener, exception);
         assertFalse(writeContext.hasQueuedWriteOps());
     }
 

From ff3c19ed13939abdf6213fadc8e75d1eed6b326b Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Wed, 29 Nov 2017 18:02:26 -0500
Subject: [PATCH 126/297] Move DNS cache settings to important configuration

This commit moves the DNS cache settings for the JVM to the important
settings section of the docs.

Relates #27592
---
 docs/reference/modules/discovery/zen.asciidoc | 16 ++++++----------
 .../setup/important-settings.asciidoc         | 19 +++++++++++++++++++
 2 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/docs/reference/modules/discovery/zen.asciidoc b/docs/reference/modules/discovery/zen.asciidoc
index 0cce897f115..f0f26a46659 100644
--- a/docs/reference/modules/discovery/zen.asciidoc
+++ b/docs/reference/modules/discovery/zen.asciidoc
@@ -22,16 +22,12 @@ other nodes.
 [[unicast]]
 ===== Unicast
 
-Unicast discovery requires a list of hosts to use that will act as gossip routers. These hosts can be specified as
-hostnames or IP addresses; hosts specified as hostnames are resolved to IP addresses during each round of pinging. Note
-that with the Java security manager in place, the JVM defaults to caching positive hostname resolutions indefinitely.
-This can be modified by adding
-http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.ttl=`] to your
-http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java security policy]. Any hosts that
-fail to resolve will be logged. Note also that with the Java security manager in place, the JVM defaults to caching
-negative hostname resolutions for ten seconds. This can be modified by adding
-http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.negative.ttl=`]
-to your http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java security policy].
+Unicast discovery requires a list of hosts to use that will act as gossip
+routers. These hosts can be specified as hostnames or IP addresses; hosts
+specified as hostnames are resolved to IP addresses during each round of
+pinging. Note that if you are in an environment where DNS resolutions vary with
+time, you might need to adjust your <>.
 
 It is recommended that the unicast hosts list be maintained as the list of
 master-eligible nodes in the cluster.
diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc
index aa86e9be268..c45839787b5 100644
--- a/docs/reference/setup/important-settings.asciidoc
+++ b/docs/reference/setup/important-settings.asciidoc
@@ -201,3 +201,22 @@ the Elasticsearch process. If you wish to configure a heap dump path, you should
 modify the entry `#-XX:HeapDumpPath=/heap/dump/path` in
 <> to remove the comment marker `#` and to specify an
 actual path.
+
+[float]
+[[networkaddress-cache-ttl]]
+
+Elasticsearch runs with a security manager in place. With a security manager in
+place, the JVM defaults to caching positive hostname resolutions
+indefinitely. If your Elasticsearch nodes rely on DNS in an environment where
+DNS resolutions vary with time (e.g., for node-to-node discovery) then you might
+want to modify the default JVM behavior.  This can be modified by adding
+http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.ttl=`]
+to your
+http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java
+security policy]. Any hosts that fail to resolve will be logged. Note also that
+with the Java security manager in place, the JVM defaults to caching negative
+hostname resolutions for ten seconds. This can be modified by adding
+http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.negative.ttl=`]
+to your
+http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java
+security policy].

From 6655689b15f6ba2d5e94ae7640897e722f877d9d Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Wed, 29 Nov 2017 19:57:26 -0500
Subject: [PATCH 127/297] Move DNS cache docs to system configuration docs

When these docs were moved they should have been moved to the system
configuration docs. This commit does that, and also fixes a missing
heading that broke the docs build.
---
 .../setup/important-settings.asciidoc         | 19 -------------------
 docs/reference/setup/sysconfig.asciidoc       |  3 +--
 .../setup/sysconfig/dns-cache.asciidoc        | 18 ++++++++++++++++++
 3 files changed, 19 insertions(+), 21 deletions(-)
 create mode 100644 docs/reference/setup/sysconfig/dns-cache.asciidoc

diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc
index c45839787b5..aa86e9be268 100644
--- a/docs/reference/setup/important-settings.asciidoc
+++ b/docs/reference/setup/important-settings.asciidoc
@@ -201,22 +201,3 @@ the Elasticsearch process. If you wish to configure a heap dump path, you should
 modify the entry `#-XX:HeapDumpPath=/heap/dump/path` in
 <> to remove the comment marker `#` and to specify an
 actual path.
-
-[float]
-[[networkaddress-cache-ttl]]
-
-Elasticsearch runs with a security manager in place. With a security manager in
-place, the JVM defaults to caching positive hostname resolutions
-indefinitely. If your Elasticsearch nodes rely on DNS in an environment where
-DNS resolutions vary with time (e.g., for node-to-node discovery) then you might
-want to modify the default JVM behavior.  This can be modified by adding
-http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.ttl=`]
-to your
-http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java
-security policy]. Any hosts that fail to resolve will be logged. Note also that
-with the Java security manager in place, the JVM defaults to caching negative
-hostname resolutions for ten seconds. This can be modified by adding
-http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.negative.ttl=`]
-to your
-http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java
-security policy].
diff --git a/docs/reference/setup/sysconfig.asciidoc b/docs/reference/setup/sysconfig.asciidoc
index 68a098582a4..5dc015f1451 100644
--- a/docs/reference/setup/sysconfig.asciidoc
+++ b/docs/reference/setup/sysconfig.asciidoc
@@ -41,5 +41,4 @@ include::sysconfig/virtual-memory.asciidoc[]
 
 include::sysconfig/threads.asciidoc[]
 
-
-
+include::sysconfig/dns-cache.asciidoc[]
diff --git a/docs/reference/setup/sysconfig/dns-cache.asciidoc b/docs/reference/setup/sysconfig/dns-cache.asciidoc
new file mode 100644
index 00000000000..76ef4f52042
--- /dev/null
+++ b/docs/reference/setup/sysconfig/dns-cache.asciidoc
@@ -0,0 +1,18 @@
+[[networkaddress-cache-ttl]]
+=== DNS cache settings
+
+Elasticsearch runs with a security manager in place. With a security manager in
+place, the JVM defaults to caching positive hostname resolutions
+indefinitely. If your Elasticsearch nodes rely on DNS in an environment where
+DNS resolutions vary with time (e.g., for node-to-node discovery) then you might
+want to modify the default JVM behavior.  This can be modified by adding
+http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.ttl=`]
+to your
+http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java
+security policy]. Any hosts that fail to resolve will be logged. Note also that
+with the Java security manager in place, the JVM defaults to caching negative
+hostname resolutions for ten seconds. This can be modified by adding
+http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.negative.ttl=`]
+to your
+http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java
+security policy].
\ No newline at end of file

From 64ca0fe9bb41da84945582e7317b714a642c1902 Mon Sep 17 00:00:00 2001
From: Philipp Krenn 
Date: Thu, 30 Nov 2017 02:29:06 +0000
Subject: [PATCH 128/297] Update docs regarding SHA-512 checksums

This commit updates the docs for the new SHA-512 checksums that are
supported for official plugins.

Relates #27524
---
 docs/reference/setup/install/deb.asciidoc       |  7 ++++---
 docs/reference/setup/install/rpm.asciidoc       |  7 ++++---
 docs/reference/setup/install/zip-targz.asciidoc | 14 ++++++++------
 3 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc
index 536ee35551c..da043772252 100644
--- a/docs/reference/setup/install/deb.asciidoc
+++ b/docs/reference/setup/install/deb.asciidoc
@@ -119,11 +119,12 @@ The Debian package for Elasticsearch v{version} can be downloaded from the websi
 ["source","sh",subs="attributes"]
 --------------------------------------------
 wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.deb
-sha1sum elasticsearch-{version}.deb <1>
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.deb.sha512
+shasum -a 512 -c elasticsearch-{version}.deb.sha512 <1>
 sudo dpkg -i elasticsearch-{version}.deb
 --------------------------------------------
-<1> Compare the SHA produced by `sha1sum` or `shasum` with the
-    https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.deb.sha1[published SHA].
+<1> Compares the SHA of the downloaded Debian package and the published checksum, which should output
+    `elasticsearch-{version}.deb: OK`.
 
 endif::[]
 
diff --git a/docs/reference/setup/install/rpm.asciidoc b/docs/reference/setup/install/rpm.asciidoc
index 1530187d131..b820a4f71b6 100644
--- a/docs/reference/setup/install/rpm.asciidoc
+++ b/docs/reference/setup/install/rpm.asciidoc
@@ -104,11 +104,12 @@ The RPM for Elasticsearch v{version} can be downloaded from the website and inst
 ["source","sh",subs="attributes"]
 --------------------------------------------
 wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.rpm
-sha1sum elasticsearch-{version}.rpm <1>
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.rpm.sha512
+shasum -a 512 -c elasticsearch-{version}.rpm.sha512 <1>
 sudo rpm --install elasticsearch-{version}.rpm
 --------------------------------------------
-<1> Compare the SHA produced by `sha1sum` or `shasum` with the
-    https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.rpm.sha1[published SHA].
+<1> Compares the SHA of the downloaded RPM and the published checksum, which should output
+    `elasticsearch-{version}.rpm: OK`.
 
 endif::[]
 
diff --git a/docs/reference/setup/install/zip-targz.asciidoc b/docs/reference/setup/install/zip-targz.asciidoc
index 9f95a195cf3..ac7470d1238 100644
--- a/docs/reference/setup/install/zip-targz.asciidoc
+++ b/docs/reference/setup/install/zip-targz.asciidoc
@@ -31,12 +31,13 @@ The `.zip` archive for Elasticsearch v{version} can be downloaded and installed
 ["source","sh",subs="attributes"]
 --------------------------------------------
 wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip
-sha1sum elasticsearch-{version}.zip <1>
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip.sha512
+shasum -a 512 -c elasticsearch-{version}.zip.sha512 <1>
 unzip elasticsearch-{version}.zip
 cd elasticsearch-{version}/ <2>
 --------------------------------------------
-<1> Compare the SHA produced by `sha1sum` or `shasum` with the
-    https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip.sha1[published SHA].
+<1> Compares the SHA of the downloaded `.zip` archive and the published checksum, which should output
+    `elasticsearch-{version}.zip: OK`.
 <2> This directory is known as `$ES_HOME`.
 
 endif::[]
@@ -58,12 +59,13 @@ The `.tar.gz` archive for Elasticsearch v{version} can be downloaded and install
 ["source","sh",subs="attributes"]
 --------------------------------------------
 wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz
-sha1sum elasticsearch-{version}.tar.gz <1>
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz.sha512
+shasum -a 512 -c elasticsearch-{version}.tar.gz.sha512 <1>
 tar -xzf elasticsearch-{version}.tar.gz
 cd elasticsearch-{version}/ <2>
 --------------------------------------------
-<1> Compare the SHA produced by `sha1sum` or `shasum` with the
-    https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz.sha1[published SHA].
+<1> Compares the SHA of the downloaded `.tar.gz` archive and the published checksum, which should output
+    `elasticsearch-{version}.tar.gz: OK`.
 <2> This directory is known as `$ES_HOME`.
 
 endif::[]

From 2738c783e57650fe57531a2a916a2ae6929b3a3f Mon Sep 17 00:00:00 2001
From: Yannick Welsch 
Date: Thu, 30 Nov 2017 10:03:16 +0100
Subject: [PATCH 129/297] Fix Gradle >=4.2 compatibility (#27591)

Gradle 4.2 introduced a feature for safer handling of stale output files. Unfortunately, due to the way some of our tasks are written, this broke execution of our REST tests tasks. The reason for this is that the extract task (which extracts the ES distribution) would clean its output directory, and thereby also remove the empty cwd subdirectory which was created by the clean task. The reason why Gradle would remove the directory is that the earlier running clean task would programmatically create the empty cwd directory, but not make Gradle aware of this fact, which would result in Gradle believing that it could just safely clean the directory.

This commit explicitly defines a task to create the cwd subdirectory, and marks the directory as output of the task, so that the subsequent extract task won't eagerly clean it. It thereby restores full compatibility of the build with Gradle 4.2 and above.

This commit also removes the @Input annotation on the waitCondition closure of AntFixture, which conflicted with the extended input/output validation of Gradle 4.3.
---
 .../main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 6 ------
 .../groovy/org/elasticsearch/gradle/test/AntFixture.groovy  | 1 -
 .../elasticsearch/gradle/test/ClusterFormationTasks.groovy  | 6 +++++-
 3 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index add518822e0..c1669c0a72a 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -131,12 +131,6 @@ class BuildPlugin implements Plugin {
                 throw new GradleException("${minGradle} or above is required to build elasticsearch")
             }
 
-            final GradleVersion gradle42 = GradleVersion.version('4.2')
-            final GradleVersion gradle43 = GradleVersion.version('4.3')
-            if (currentGradleVersion >= gradle42 && currentGradleVersion < gradle43) {
-                throw new GradleException("${currentGradleVersion} is not compatible with the elasticsearch build")
-            }
-
             // enforce Java version
             if (javaVersionEnum < minimumJava) {
                 throw new GradleException("Java ${minimumJava} or above is required to build Elasticsearch")
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy
index 34c3046aa2b..039bce05226 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy
@@ -69,7 +69,6 @@ public class AntFixture extends AntTask implements Fixture {
      * as well as a groovy AntBuilder, to enable running ant condition checks. The default wait
      * condition is for http on the http port.
      */
-    @Input
     Closure waitCondition = { AntFixture fixture, AntBuilder ant ->
         File tmpFile = new File(fixture.cwd, 'wait.success')
         ant.get(src: "http://${fixture.addressAndPort}",
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 72c6eef6685..f7637d23366 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -166,11 +166,13 @@ class ClusterFormationTasks {
         Task setup = project.tasks.create(name: taskName(prefix, node, 'clean'), type: Delete, dependsOn: dependsOn) {
             delete node.homeDir
             delete node.cwd
+        }
+        setup = project.tasks.create(name: taskName(prefix, node, 'createCwd'), type: DefaultTask, dependsOn: setup) {
             doLast {
                 node.cwd.mkdirs()
             }
+            outputs.dir node.cwd
         }
-
         setup = configureCheckPreviousTask(taskName(prefix, node, 'checkPrevious'), project, setup, node)
         setup = configureStopTask(taskName(prefix, node, 'stopPrevious'), project, setup, node)
         setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distribution)
@@ -281,6 +283,7 @@ class ClusterFormationTasks {
                         rpmDatabase.deleteDir()
                         rpmExtracted.deleteDir()
                     }
+                    outputs.dir rpmExtracted
                 }
                 break;
             case 'deb':
@@ -292,6 +295,7 @@ class ClusterFormationTasks {
                     doFirst {
                         debExtracted.deleteDir()
                     }
+                    outputs.dir debExtracted
                 }
                 break;
             default:

From 192d1f03f803392e4e9771a9fc76e7322be8e00a Mon Sep 17 00:00:00 2001
From: Tanguy Leroux 
Date: Thu, 30 Nov 2017 10:09:49 +0100
Subject: [PATCH 130/297] Do not swallow exception in
 ChecksumBlobStoreFormat.writeAtomic() (#27597)

The ChecksumBlobStoreFormat.writeAtomic() method writes a blob using a
temporary name and then moves the blob to its final name. The move
operation can fail and in this case the temporary blob is deleted. If
this delete operation also fails, then the initial exception is lost.

This commit ensures that when something goes wrong during the move
operation the initial exception is kept and thrown, and if the delete
operation also fails then this additional exception is added
as a suppressed exception to the initial one.
---
 .../blobstore/ChecksumBlobStoreFormat.java    |  6 +-
 .../snapshots/BlobStoreFormatIT.java          | 62 +++++++++++++++++++
 2 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
index 0cb38d9976d..a959cd0efb8 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
@@ -138,7 +138,11 @@ public class ChecksumBlobStoreFormat extends BlobStoreForm
             blobContainer.move(tempBlobName, blobName);
         } catch (IOException ex) {
             // Move failed - try cleaning up
-            blobContainer.deleteBlob(tempBlobName);
+            try {
+                blobContainer.deleteBlob(tempBlobName);
+            } catch (Exception e) {
+                ex.addSuppressed(e);
+            }
             throw ex;
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
index 5c1fe5d2c2c..65926234d45 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
@@ -38,6 +38,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.translog.BufferedChecksumStreamOutput;
 import org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat;
+import org.elasticsearch.snapshots.mockstore.BlobContainerWrapper;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -210,6 +211,67 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
         }
     }
 
+    public void testAtomicWriteFailures() throws Exception {
+        final String name = randomAlphaOfLength(10);
+        final BlobObj blobObj = new BlobObj("test");
+        final ChecksumBlobStoreFormat checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
+
+        final BlobStore blobStore = createTestBlobStore();
+        final BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
+
+        {
+            IOException writeBlobException = expectThrows(IOException.class, () -> {
+                BlobContainer wrapper = new BlobContainerWrapper(blobContainer) {
+                    @Override
+                    public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
+                        throw new IOException("Exception thrown in writeBlob() for " + blobName);
+                    }
+                };
+                checksumFormat.writeAtomic(blobObj, wrapper, name);
+            });
+
+            assertEquals("Exception thrown in writeBlob() for pending-" + name, writeBlobException.getMessage());
+            assertEquals(0, writeBlobException.getSuppressed().length);
+        }
+        {
+            IOException moveException = expectThrows(IOException.class, () -> {
+                BlobContainer wrapper = new BlobContainerWrapper(blobContainer) {
+                    @Override
+                    public void move(String sourceBlobName, String targetBlobName) throws IOException {
+                        throw new IOException("Exception thrown in move() for " + sourceBlobName);
+                    }
+                };
+                checksumFormat.writeAtomic(blobObj, wrapper, name);
+            });
+            assertEquals("Exception thrown in move() for pending-" + name, moveException.getMessage());
+            assertEquals(0, moveException.getSuppressed().length);
+        }
+        {
+            IOException moveThenDeleteException = expectThrows(IOException.class, () -> {
+                BlobContainer wrapper = new BlobContainerWrapper(blobContainer) {
+                    @Override
+                    public void move(String sourceBlobName, String targetBlobName) throws IOException {
+                        throw new IOException("Exception thrown in move() for " + sourceBlobName);
+                    }
+
+                    @Override
+                    public void deleteBlob(String blobName) throws IOException {
+                        throw new IOException("Exception thrown in deleteBlob() for " + blobName);
+                    }
+                };
+                checksumFormat.writeAtomic(blobObj, wrapper, name);
+            });
+
+            assertEquals("Exception thrown in move() for pending-" + name, moveThenDeleteException.getMessage());
+            assertEquals(1, moveThenDeleteException.getSuppressed().length);
+
+            final Throwable suppressedThrowable = moveThenDeleteException.getSuppressed()[0];
+            assertTrue(suppressedThrowable instanceof IOException);
+            assertEquals("Exception thrown in deleteBlob() for pending-" + name, suppressedThrowable.getMessage());
+        }
+    }
+
     protected BlobStore createTestBlobStore() throws IOException {
         Settings settings = Settings.builder().build();
         return new FsBlobStore(settings, randomRepoPath());

From efac982e352164f40851a9ff063f7290bf267a0d Mon Sep 17 00:00:00 2001
From: kel 
Date: Thu, 30 Nov 2017 17:38:07 +0800
Subject: [PATCH 131/297] Include include_global_state in Snapshot status API
 (#26853)

This commit adds a field include_global_state to snapshot status api response. For legacy snapshot, the field is not present.

Closes #22423
---
 .../snapshots/status/SnapshotStatus.java      | 27 +++++++++++-
 .../TransportSnapshotsStatusAction.java       |  6 ++-
 .../repositories/Repository.java              |  3 +-
 .../blobstore/BlobStoreRepository.java        |  7 +--
 .../elasticsearch/snapshots/SnapshotInfo.java | 44 +++++++++++++++----
 .../snapshots/SnapshotsService.java           |  8 ++--
 .../snapshots/status/SnapshotStatusTests.java |  4 +-
 .../index/shard/IndexShardTests.java          |  5 ++-
 .../SharedClusterSnapshotRestoreIT.java       | 17 ++++++-
 9 files changed, 99 insertions(+), 22 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
index e8c797e45a0..a4613af601a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
@@ -19,8 +19,10 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.status;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.SnapshotsInProgress.State;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -57,10 +59,15 @@ public class SnapshotStatus implements ToXContentObject, Streamable {
 
     private SnapshotStats stats;
 
-    SnapshotStatus(final Snapshot snapshot, final State state, final List shards) {
+    @Nullable
+    private Boolean includeGlobalState;
+
+    SnapshotStatus(final Snapshot snapshot, final State state, final List shards,
+                   final Boolean includeGlobalState) {
         this.snapshot = Objects.requireNonNull(snapshot);
         this.state = Objects.requireNonNull(state);
         this.shards = Objects.requireNonNull(shards);
+        this.includeGlobalState = includeGlobalState;
         shardsStats = new SnapshotShardsStats(shards);
         updateShardStats();
     }
@@ -82,6 +89,14 @@ public class SnapshotStatus implements ToXContentObject, Streamable {
         return state;
     }
 
+    /**
+     * Returns true if global state is included in the snapshot, false otherwise.
+     * Can be null if this information is unknown.
+     */
+    public Boolean includeGlobalState() {
+        return includeGlobalState;
+    }
+
     /**
      * Returns list of snapshot shards
      */
@@ -132,6 +147,9 @@ public class SnapshotStatus implements ToXContentObject, Streamable {
             builder.add(SnapshotIndexShardStatus.readShardSnapshotStatus(in));
         }
         shards = Collections.unmodifiableList(builder);
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            includeGlobalState = in.readOptionalBoolean();
+        }
         updateShardStats();
     }
 
@@ -143,6 +161,9 @@ public class SnapshotStatus implements ToXContentObject, Streamable {
         for (SnapshotIndexShardStatus shard : shards) {
             shard.writeTo(out);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeOptionalBoolean(includeGlobalState);
+        }
     }
 
     /**
@@ -174,6 +195,7 @@ public class SnapshotStatus implements ToXContentObject, Streamable {
     private static final String UUID = "uuid";
     private static final String STATE = "state";
     private static final String INDICES = "indices";
+    private static final String INCLUDE_GLOBAL_STATE = "include_global_state";
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
@@ -182,6 +204,9 @@ public class SnapshotStatus implements ToXContentObject, Streamable {
         builder.field(REPOSITORY, snapshot.getRepository());
         builder.field(UUID, snapshot.getSnapshotId().getUUID());
         builder.field(STATE, state.name());
+        if (includeGlobalState != null) {
+            builder.field(INCLUDE_GLOBAL_STATE, includeGlobalState);
+        }
         shardsStats.toXContent(builder, params);
         stats.toXContent(builder, params);
         builder.startObject(INDICES);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index 88884fd4395..71bb1995dd5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -196,7 +196,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction indices, long startTime, String failure, int totalShards,
-                                  List shardFailures, long repositoryStateId);
+                                  List shardFailures, long repositoryStateId, boolean includeGlobalState);
 
     /**
      * Deletes snapshot
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 440867577ea..a66a5a51d10 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -468,11 +468,12 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                                          final String failure,
                                          final int totalShards,
                                          final List shardFailures,
-                                         final long repositoryStateId) {
-
+                                         final long repositoryStateId,
+                                         final boolean includeGlobalState) {
         SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId,
             indices.stream().map(IndexId::getName).collect(Collectors.toList()),
-            startTime, failure, System.currentTimeMillis(), totalShards, shardFailures);
+            startTime, failure, System.currentTimeMillis(), totalShards, shardFailures,
+            includeGlobalState);
         try {
             snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, snapshotId.getUUID());
             final RepositoryData repositoryData = getRepositoryData();
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index de0a52ed0e4..ea2a9ac78dd 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -69,8 +69,10 @@ public final class SnapshotInfo implements Comparable, ToXContent,
     private static final String NAME = "name";
     private static final String TOTAL_SHARDS = "total_shards";
     private static final String SUCCESSFUL_SHARDS = "successful_shards";
+    private static final String INCLUDE_GLOBAL_STATE = "include_global_state";
 
     private static final Version VERSION_INCOMPATIBLE_INTRODUCED = Version.V_5_2_0;
+    private static final Version INCLUDE_GLOBAL_STATE_INTRODUCED = Version.V_7_0_0_alpha1;
     public static final Version VERBOSE_INTRODUCED = Version.V_5_5_0;
 
     private static final Comparator COMPARATOR =
@@ -94,27 +96,32 @@ public final class SnapshotInfo implements Comparable, ToXContent,
 
     private final int successfulShards;
 
+    @Nullable
+    private Boolean includeGlobalState;
+
     @Nullable
     private final Version version;
 
     private final List shardFailures;
 
     public SnapshotInfo(SnapshotId snapshotId, List indices, SnapshotState state) {
-        this(snapshotId, indices, state, null, null, 0L, 0L, 0, 0, Collections.emptyList());
+        this(snapshotId, indices, state, null, null, 0L, 0L, 0, 0, Collections.emptyList(), null);
     }
 
-    public SnapshotInfo(SnapshotId snapshotId, List indices, long startTime) {
-        this(snapshotId, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, 0, 0, Collections.emptyList());
+    public SnapshotInfo(SnapshotId snapshotId, List indices, long startTime, Boolean includeGlobalState) {
+        this(snapshotId, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, 0, 0,
+            Collections.emptyList(), includeGlobalState);
     }
 
     public SnapshotInfo(SnapshotId snapshotId, List indices, long startTime, String reason, long endTime,
-                        int totalShards, List shardFailures) {
+                        int totalShards, List shardFailures, Boolean includeGlobalState) {
         this(snapshotId, indices, snapshotState(reason, shardFailures), reason, Version.CURRENT,
-             startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures);
+             startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures, includeGlobalState);
     }
 
     private SnapshotInfo(SnapshotId snapshotId, List indices, SnapshotState state, String reason, Version version,
-                         long startTime, long endTime, int totalShards, int successfulShards, List shardFailures) {
+                         long startTime, long endTime, int totalShards, int successfulShards, List shardFailures,
+                         Boolean includeGlobalState) {
         this.snapshotId = Objects.requireNonNull(snapshotId);
         this.indices = Collections.unmodifiableList(Objects.requireNonNull(indices));
         this.state = state;
@@ -125,6 +132,7 @@ public final class SnapshotInfo implements Comparable, ToXContent,
         this.totalShards = totalShards;
         this.successfulShards = successfulShards;
         this.shardFailures = Objects.requireNonNull(shardFailures);
+        this.includeGlobalState = includeGlobalState;
     }
 
     /**
@@ -163,6 +171,9 @@ public final class SnapshotInfo implements Comparable, ToXContent,
         } else {
             version = in.readBoolean() ? Version.readVersion(in) : null;
         }
+        if (in.getVersion().onOrAfter(INCLUDE_GLOBAL_STATE_INTRODUCED)) {
+            includeGlobalState = in.readOptionalBoolean();
+        }
     }
 
     /**
@@ -172,7 +183,7 @@ public final class SnapshotInfo implements Comparable, ToXContent,
     public static SnapshotInfo incompatible(SnapshotId snapshotId) {
         return new SnapshotInfo(snapshotId, Collections.emptyList(), SnapshotState.INCOMPATIBLE,
                                 "the snapshot is incompatible with the current version of Elasticsearch and its exact version is unknown",
-                                null, 0L, 0L, 0, 0, Collections.emptyList());
+                                null, 0L, 0L, 0, 0, Collections.emptyList(), null);
     }
 
     /**
@@ -271,6 +282,10 @@ public final class SnapshotInfo implements Comparable, ToXContent,
         return successfulShards;
     }
 
+    public Boolean includeGlobalState() {
+        return includeGlobalState;
+    }
+
     /**
      * Returns shard failures; an empty list will be returned if there were no shard
      * failures, or if {@link #state()} returns {@code null}.
@@ -361,6 +376,9 @@ public final class SnapshotInfo implements Comparable, ToXContent,
             builder.value(index);
         }
         builder.endArray();
+        if (includeGlobalState != null) {
+            builder.field(INCLUDE_GLOBAL_STATE, includeGlobalState);
+        }
         if (verbose || state != null) {
             builder.field(STATE, state);
         }
@@ -411,6 +429,9 @@ public final class SnapshotInfo implements Comparable, ToXContent,
         if (reason != null) {
             builder.field(REASON, reason);
         }
+        if (includeGlobalState != null) {
+            builder.field(INCLUDE_GLOBAL_STATE, includeGlobalState);
+        }
         builder.field(START_TIME, startTime);
         builder.field(END_TIME, endTime);
         builder.field(TOTAL_SHARDS, totalShards);
@@ -442,6 +463,7 @@ public final class SnapshotInfo implements Comparable, ToXContent,
         long endTime = 0;
         int totalShards = 0;
         int successfulShards = 0;
+        Boolean includeGlobalState = null;
         List shardFailures = Collections.emptyList();
         if (parser.currentToken() == null) { // fresh parser? move to the first token
             parser.nextToken();
@@ -476,6 +498,8 @@ public final class SnapshotInfo implements Comparable, ToXContent,
                                 successfulShards = parser.intValue();
                             } else if (VERSION_ID.equals(currentFieldName)) {
                                 version = Version.fromId(parser.intValue());
+                            } else if (INCLUDE_GLOBAL_STATE.equals(currentFieldName)) {
+                                includeGlobalState = parser.booleanValue();
                             }
                         } else if (token == XContentParser.Token.START_ARRAY) {
                             if (INDICES.equals(currentFieldName)) {
@@ -517,7 +541,8 @@ public final class SnapshotInfo implements Comparable, ToXContent,
                                 endTime,
                                 totalShards,
                                 successfulShards,
-                                shardFailures);
+                                shardFailures,
+                                includeGlobalState);
     }
 
     @Override
@@ -564,6 +589,9 @@ public final class SnapshotInfo implements Comparable, ToXContent,
                 out.writeBoolean(false);
             }
         }
+        if (out.getVersion().onOrAfter(INCLUDE_GLOBAL_STATE_INTRODUCED)) {
+            out.writeOptionalBoolean(includeGlobalState);
+        }
     }
 
     private static SnapshotState snapshotState(final String reason, final List shardFailures) {
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 0804e69e46e..5092a58adaa 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -496,7 +496,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                                                          ExceptionsHelper.detailedMessage(exception),
                                                          0,
                                                          Collections.emptyList(),
-                                                         snapshot.getRepositoryStateId());
+                                                         snapshot.getRepositoryStateId(),
+                                                         snapshot.includeGlobalState());
                 } catch (Exception inner) {
                     inner.addSuppressed(exception);
                     logger.warn((Supplier) () -> new ParameterizedMessage("[{}] failed to close snapshot in repository", snapshot.snapshot()), inner);
@@ -510,7 +511,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
     private SnapshotInfo inProgressSnapshot(SnapshotsInProgress.Entry entry) {
         return new SnapshotInfo(entry.snapshot().getSnapshotId(),
                                    entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()),
-                                   entry.startTime());
+                                   entry.startTime(), entry.includeGlobalState());
     }
 
     /**
@@ -968,7 +969,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                         failure,
                         entry.shards().size(),
                         Collections.unmodifiableList(shardFailures),
-                        entry.getRepositoryStateId());
+                        entry.getRepositoryStateId(),
+                        entry.includeGlobalState());
                     removeSnapshotFromClusterState(snapshot, snapshotInfo, null);
                 } catch (Exception e) {
                     logger.warn((Supplier) () -> new ParameterizedMessage("[{}] failed to finalize snapshot", snapshot), e);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java
index 481bf5579e2..5c386174610 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java
@@ -46,7 +46,8 @@ public class SnapshotStatusTests extends ESTestCase {
         SnapshotIndexShardStatus snapshotIndexShardStatus = new SnapshotIndexShardStatus(testShardId, shardStage);
         List snapshotIndexShardStatuses = new ArrayList<>();
         snapshotIndexShardStatuses.add(snapshotIndexShardStatus);
-        SnapshotStatus status = new SnapshotStatus(snapshot, state, snapshotIndexShardStatuses);
+        boolean includeGlobalState = randomBoolean();
+        SnapshotStatus status = new SnapshotStatus(snapshot, state, snapshotIndexShardStatuses, includeGlobalState);
 
         int initializingShards = 0;
         int startedShards = 0;
@@ -80,6 +81,7 @@ public class SnapshotStatusTests extends ESTestCase {
             "  \"repository\" : \"test-repo\",\n" +
             "  \"uuid\" : \"" + uuid + "\",\n" +
             "  \"state\" : \"" + state.toString() + "\",\n" +
+            "  \"include_global_state\" : " + includeGlobalState + ",\n" +
             "  \"shards_stats\" : {\n" +
             "    \"initializing\" : " + initializingShards + ",\n" +
             "    \"started\" : " + startedShards + ",\n" +
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index fe6d4ccdea0..00f29dc252d 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -2542,8 +2542,9 @@ public class IndexShardTests extends IndexShardTestCase {
         }
 
         @Override
-        public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List indices, long startTime, String failure, int totalShards,
-                                             List shardFailures, long repositoryStateId) {
+        public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List indices, long startTime, String failure,
+                                             int totalShards, List shardFailures, long repositoryStateId,
+                                             boolean includeGlobalState) {
             return null;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 6a6825a8ada..fa3920c1e8d 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -590,12 +590,20 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0));
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-no-global-state").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+        SnapshotsStatusResponse snapshotsStatusResponse = client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap-no-global-state").get();
+        assertThat(snapshotsStatusResponse.getSnapshots().size(), equalTo(1));
+        SnapshotStatus snapshotStatus = snapshotsStatusResponse.getSnapshots().get(0);
+        assertThat(snapshotStatus.includeGlobalState(), equalTo(false));
 
         logger.info("--> snapshot with global state");
         createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-with-global-state").setIndices().setIncludeGlobalState(true).setWaitForCompletion(true).get();
         assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0));
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-with-global-state").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+        snapshotsStatusResponse = client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap-with-global-state").get();
+        assertThat(snapshotsStatusResponse.getSnapshots().size(), equalTo(1));
+        snapshotStatus = snapshotsStatusResponse.getSnapshots().get(0);
+        assertThat(snapshotStatus.includeGlobalState(), equalTo(true));
 
         if (testTemplate) {
             logger.info("-->  delete test template");
@@ -1635,7 +1643,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         String blockedNode = blockNodeWithIndex("test-repo", "test-idx");
 
         logger.info("--> snapshot");
-        client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get();
+        client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
+            .setWaitForCompletion(false).setIncludeGlobalState(false).setIndices("test-idx").get();
 
         logger.info("--> waiting for block to kick in");
         waitForBlock(blockedNode, "test-repo", TimeValue.timeValueSeconds(60));
@@ -1645,6 +1654,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(response.getSnapshots().size(), equalTo(1));
         SnapshotStatus snapshotStatus = response.getSnapshots().get(0);
         assertThat(snapshotStatus.getState(), equalTo(SnapshotsInProgress.State.STARTED));
+        assertThat(snapshotStatus.includeGlobalState(), equalTo(false));
+
         // We blocked the node during data write operation, so at least one shard snapshot should be in STARTED stage
         assertThat(snapshotStatus.getShardsStats().getStartedShards(), greaterThan(0));
         for (SnapshotIndexShardStatus shardStatus : snapshotStatus.getIndices().get("test-idx")) {
@@ -1658,6 +1669,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(response.getSnapshots().size(), equalTo(1));
         snapshotStatus = response.getSnapshots().get(0);
         assertThat(snapshotStatus.getState(), equalTo(SnapshotsInProgress.State.STARTED));
+        assertThat(snapshotStatus.includeGlobalState(), equalTo(false));
+
         // We blocked the node during data write operation, so at least one shard snapshot should be in STARTED stage
         assertThat(snapshotStatus.getShardsStats().getStartedShards(), greaterThan(0));
         for (SnapshotIndexShardStatus shardStatus : snapshotStatus.getIndices().get("test-idx")) {
@@ -1684,6 +1697,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         response = client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap").execute().actionGet();
         snapshotStatus = response.getSnapshots().get(0);
         assertThat(snapshotStatus.getIndices().size(), equalTo(1));
+        assertThat(snapshotStatus.includeGlobalState(), equalTo(false));
+
         SnapshotIndexStatus indexStatus = snapshotStatus.getIndices().get("test-idx");
         assertThat(indexStatus, notNullValue());
         assertThat(indexStatus.getShardsStats().getInitializingShards(), equalTo(0));

From 41f73e0acf7376c8a33e1747c31dd553bea27d5e Mon Sep 17 00:00:00 2001
From: Tanguy Leroux 
Date: Thu, 30 Nov 2017 11:33:01 +0100
Subject: [PATCH 132/297] Fix version for include_global_state in Snapshot
 Status API

It also adds a Rest test.

Related #26853
---
 .../snapshots/status/SnapshotStatus.java      |  4 +-
 .../elasticsearch/snapshots/SnapshotInfo.java |  2 +-
 .../test/snapshot.get/10_basic.yml            | 62 ++++++++++++++++++-
 3 files changed, 64 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
index a4613af601a..7b41d96c0e3 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
@@ -147,7 +147,7 @@ public class SnapshotStatus implements ToXContentObject, Streamable {
             builder.add(SnapshotIndexShardStatus.readShardSnapshotStatus(in));
         }
         shards = Collections.unmodifiableList(builder);
-        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+        if (in.getVersion().onOrAfter(Version.V_6_2_0)) {
             includeGlobalState = in.readOptionalBoolean();
         }
         updateShardStats();
@@ -161,7 +161,7 @@ public class SnapshotStatus implements ToXContentObject, Streamable {
         for (SnapshotIndexShardStatus shard : shards) {
             shard.writeTo(out);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+        if (out.getVersion().onOrAfter(Version.V_6_2_0)) {
             out.writeOptionalBoolean(includeGlobalState);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index ea2a9ac78dd..b75b840e8b2 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -72,7 +72,7 @@ public final class SnapshotInfo implements Comparable, ToXContent,
     private static final String INCLUDE_GLOBAL_STATE = "include_global_state";
 
     private static final Version VERSION_INCOMPATIBLE_INTRODUCED = Version.V_5_2_0;
-    private static final Version INCLUDE_GLOBAL_STATE_INTRODUCED = Version.V_7_0_0_alpha1;
+    private static final Version INCLUDE_GLOBAL_STATE_INTRODUCED = Version.V_6_2_0;
     public static final Version VERBOSE_INTRODUCED = Version.V_5_5_0;
 
     private static final Comparator COMPARATOR =
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml
index 70008415122..d17655daad2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml
@@ -49,7 +49,7 @@ setup:
         snapshot: test_nonexistent_snapshot
 
 ---
-"Get missing snapshot info succeeds when ignoreUnavailable is true":
+"Get missing snapshot info succeeds when ignore_unavailable is true":
 
   - do:
       snapshot.get:
@@ -96,3 +96,63 @@ setup:
       snapshot.delete:
         repository: test_repo_get_1
         snapshot: test_snapshot
+
+---
+"Get snapshot info contains include_global_state":
+  - skip:
+      version: " - 6.1.99"
+      reason: "include_global_state field has been added in the response in 6.2.0"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            number_of_shards:   1
+            number_of_replicas: 0
+
+  - do:
+      snapshot.create:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_include_global_state
+        wait_for_completion: true
+        body: |
+          { "include_global_state": true }
+
+  - do:
+      snapshot.get:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_include_global_state
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: test_snapshot_with_include_global_state }
+  - match: { snapshots.0.state: SUCCESS }
+  - match: { snapshots.0.include_global_state: true }
+
+  - do:
+      snapshot.delete:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_include_global_state
+
+  - do:
+      snapshot.create:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_without_include_global_state
+        wait_for_completion: true
+        body: |
+          { "include_global_state": false }
+
+  - do:
+      snapshot.get:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_without_include_global_state
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: test_snapshot_without_include_global_state }
+  - match: { snapshots.0.state: SUCCESS }
+  - match: { snapshots.0.include_global_state: false }
+
+  - do:
+      snapshot.delete:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_without_include_global_state

From d25c9671de76ec1d8f1da3987b5844449403d02d Mon Sep 17 00:00:00 2001
From: olcbean 
Date: Thu, 30 Nov 2017 13:49:34 +0100
Subject: [PATCH 133/297] Deprecate `jarowinkler` in favor of `jaro_winkler`
 (#27526)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Jaro and Winkler are two people, so we should use the same naming convention as for Damerau–Levenshtein.
---
 .../phrase/DirectCandidateGeneratorBuilder.java     |  7 ++++---
 .../search/suggest/term/TermSuggestionBuilder.java  |  9 ++++++---
 .../phrase/DirectCandidateGeneratorTests.java       | 10 ++++++++--
 .../suggest/term/StringDistanceImplTests.java       | 13 +++++++++----
 .../suggest/term/TermSuggestionBuilderTests.java    |  2 +-
 .../search/suggesters/term-suggest.asciidoc         |  2 +-
 6 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
index b5d4ff716e5..9631b37e2cd 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
@@ -218,7 +218,7 @@ public final class DirectCandidateGeneratorBuilder implements CandidateGenerator
      * based on Damerau-Levenshtein algorithm.
      * 
  • levenshtein - String distance algorithm based on
      * Levenshtein edit distance algorithm.
-     * 
  • jarowinkler - String distance algorithm based on
+     * 
  • jaro_winkler - String distance algorithm based on
      * Jaro-Winkler algorithm.
      * 
  • ngram - String distance algorithm based on character
      * n-grams.
@@ -474,9 +474,10 @@ public final class DirectCandidateGeneratorBuilder implements CandidateGenerator
             return new LevensteinDistance();
         } else if ("levenshtein".equals(distanceVal)) {
             return new LevensteinDistance();
-            // TODO Jaro and Winkler are 2 people - so apply same naming logic
-            // as damerau_levenshtein
         } else if ("jarowinkler".equals(distanceVal)) {
+            DEPRECATION_LOGGER.deprecated("Deprecated distance [jarowinkler] used, replaced by [jaro_winkler]");
+            return new JaroWinklerDistance();
+        } else if ("jaro_winkler".equals(distanceVal)) {
             return new JaroWinklerDistance();
         } else if ("ngram".equals(distanceVal)) {
             return new NGramDistance();
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
index e8c8d6c3ae1..dfaf77d2337 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
@@ -221,7 +221,7 @@ public class TermSuggestionBuilder extends SuggestionBuilderlevenshtein - String distance algorithm based on
      * Levenshtein edit distance algorithm.
-     * 
  • jarowinkler - String distance algorithm based on
+     * 
  • jaro_winkler - String distance algorithm based on
      * Jaro-Winkler algorithm.
      * 
  • ngram - String distance algorithm based on character
      * n-grams.
@@ -556,7 +556,7 @@ public class TermSuggestionBuilder extends SuggestionBuilder DirectCandidateGeneratorBuilder.resolveDistance("doesnt_exist"));
@@ -88,6 +88,11 @@ public class DirectCandidateGeneratorTests extends ESTestCase {
         assertWarnings("Deprecated distance [levenstein] used, replaced by [levenshtein]");
     }
 
+    public void testJaroWinklerDeprecation() {
+        assertThat(DirectCandidateGeneratorBuilder.resolveDistance("jaroWinkler"), instanceOf(JaroWinklerDistance.class));
+        assertWarnings("Deprecated distance [jarowinkler] used, replaced by [jaro_winkler]");
+    }
+
     private static DirectCandidateGeneratorBuilder mutate(DirectCandidateGeneratorBuilder original) throws IOException {
         DirectCandidateGeneratorBuilder mutation = copy(original);
         List> mutators = new ArrayList<>();
@@ -212,7 +217,8 @@ public class DirectCandidateGeneratorTests extends ESTestCase {
         maybeSet(generator::postFilter, randomAlphaOfLengthBetween(1, 20));
         maybeSet(generator::size, randomIntBetween(1, 20));
         maybeSet(generator::sort, randomFrom("score", "frequency"));
-        maybeSet(generator::stringDistance, randomFrom("internal", "damerau_levenshtein", "levenshtein", "jarowinkler", "ngram"));
+        maybeSet(generator::stringDistance,
+                randomFrom("internal", "damerau_levenshtein", "levenshtein", "jaro_winkler", "ngram"));
         maybeSet(generator::suggestMode, randomFrom("missing", "popular", "always"));
         return generator;
     }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/term/StringDistanceImplTests.java b/core/src/test/java/org/elasticsearch/search/suggest/term/StringDistanceImplTests.java
index 42f1928d1dc..ff71c469651 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/term/StringDistanceImplTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/term/StringDistanceImplTests.java
@@ -39,7 +39,7 @@ public class StringDistanceImplTests extends AbstractWriteableEnumTestCase {
         assertThat(StringDistanceImpl.INTERNAL.ordinal(), equalTo(0));
         assertThat(StringDistanceImpl.DAMERAU_LEVENSHTEIN.ordinal(), equalTo(1));
         assertThat(StringDistanceImpl.LEVENSHTEIN.ordinal(), equalTo(2));
-        assertThat(StringDistanceImpl.JAROWINKLER.ordinal(), equalTo(3));
+        assertThat(StringDistanceImpl.JARO_WINKLER.ordinal(), equalTo(3));
         assertThat(StringDistanceImpl.NGRAM.ordinal(), equalTo(4));
     }
 
@@ -48,7 +48,7 @@ public class StringDistanceImplTests extends AbstractWriteableEnumTestCase {
         assertThat(StringDistanceImpl.resolve("internal"), equalTo(StringDistanceImpl.INTERNAL));
         assertThat(StringDistanceImpl.resolve("damerau_levenshtein"), equalTo(StringDistanceImpl.DAMERAU_LEVENSHTEIN));
         assertThat(StringDistanceImpl.resolve("levenshtein"), equalTo(StringDistanceImpl.LEVENSHTEIN));
-        assertThat(StringDistanceImpl.resolve("jarowinkler"), equalTo(StringDistanceImpl.JAROWINKLER));
+        assertThat(StringDistanceImpl.resolve("jaro_winkler"), equalTo(StringDistanceImpl.JARO_WINKLER));
         assertThat(StringDistanceImpl.resolve("ngram"), equalTo(StringDistanceImpl.NGRAM));
 
         final String doesntExist = "doesnt_exist";
@@ -63,12 +63,17 @@ public class StringDistanceImplTests extends AbstractWriteableEnumTestCase {
         assertWarnings("Deprecated distance [levenstein] used, replaced by [levenshtein]");
     }
 
+    public void testJaroWinklerDeprecation() {
+        assertThat(StringDistanceImpl.resolve("jaroWinkler"), equalTo(StringDistanceImpl.JARO_WINKLER));
+        assertWarnings("Deprecated distance [jarowinkler] used, replaced by [jaro_winkler]");
+    }
+
     @Override
     public void testWriteTo() throws IOException {
         assertWriteToStream(StringDistanceImpl.INTERNAL, 0);
         assertWriteToStream(StringDistanceImpl.DAMERAU_LEVENSHTEIN, 1);
         assertWriteToStream(StringDistanceImpl.LEVENSHTEIN, 2);
-        assertWriteToStream(StringDistanceImpl.JAROWINKLER, 3);
+        assertWriteToStream(StringDistanceImpl.JARO_WINKLER, 3);
         assertWriteToStream(StringDistanceImpl.NGRAM, 4);
     }
 
@@ -77,7 +82,7 @@ public class StringDistanceImplTests extends AbstractWriteableEnumTestCase {
         assertReadFromStream(0, StringDistanceImpl.INTERNAL);
         assertReadFromStream(1, StringDistanceImpl.DAMERAU_LEVENSHTEIN);
         assertReadFromStream(2, StringDistanceImpl.LEVENSHTEIN);
-        assertReadFromStream(3, StringDistanceImpl.JAROWINKLER);
+        assertReadFromStream(3, StringDistanceImpl.JARO_WINKLER);
         assertReadFromStream(4, StringDistanceImpl.NGRAM);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
index 1e18fc1253b..d4292151a71 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
@@ -100,7 +100,7 @@ public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCas
             case 0: return StringDistanceImpl.INTERNAL;
             case 1: return StringDistanceImpl.DAMERAU_LEVENSHTEIN;
             case 2: return StringDistanceImpl.LEVENSHTEIN;
-            case 3: return StringDistanceImpl.JAROWINKLER;
+            case 3: return StringDistanceImpl.JARO_WINKLER;
             case 4: return StringDistanceImpl.NGRAM;
             default: throw new IllegalArgumentException("No string distance algorithm with an ordinal of " + randomVal);
         }
diff --git a/docs/reference/search/suggesters/term-suggest.asciidoc b/docs/reference/search/suggesters/term-suggest.asciidoc
index d8feeaf1760..65b5c3dd9ac 100644
--- a/docs/reference/search/suggesters/term-suggest.asciidoc
+++ b/docs/reference/search/suggesters/term-suggest.asciidoc
@@ -118,5 +118,5 @@ doesn't take the query into account that is part of request.
     Damerau-Levenshtein algorithm.
     `levenshtein` - String distance algorithm based on Levenshtein edit distance
     algorithm.
-    `jarowinkler` - String distance algorithm based on Jaro-Winkler algorithm.
+    `jaro_winkler` - String distance algorithm based on Jaro-Winkler algorithm.
     `ngram` - String distance algorithm based on character n-grams.

From 1f89e9d94e69d9a1cd480aad3f8b0a2cf68a4217 Mon Sep 17 00:00:00 2001
From: David Turner 
Date: Thu, 30 Nov 2017 13:01:22 +0000
Subject: [PATCH 134/297] Reinstate AwaitsFix

This reverts commit 29c554032356c658ecf0d9d23068a1c84e9364bb.
---
 .../action/admin/indices/shards/IndicesShardStoreRequestIT.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
index b43996ddf02..cb150e012ae 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
@@ -151,6 +151,7 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
         assertThat(shardStatuses.get(index1).size(), equalTo(2));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/12416")
     public void testCorruptedShards() throws Exception {
         String index = "test";
         internalCluster().ensureAtLeastNumDataNodes(2);

From 92a24de509f388df4f569fe9576d60f0697d3beb Mon Sep 17 00:00:00 2001
From: David Turner 
Date: Thu, 30 Nov 2017 16:34:23 +0000
Subject: [PATCH 135/297] Add more logging to testCorruptedShards to help
 investigate sporadic failures

---
 .../indices/shards/IndicesShardStoreRequestIT.java   | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
index cb150e012ae..ab66694cd96 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
@@ -60,7 +60,8 @@ import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
-@TestLogging("_root:DEBUG,org.elasticsearch.action.admin.indices.shards:TRACE,org.elasticsearch.cluster.service:TRACE")
+@TestLogging("_root:DEBUG,org.elasticsearch.action.admin.indices.shards:TRACE,org.elasticsearch.cluster.service:TRACE," +
+    "org.elasticsearch.gateway.TransportNodesListGatewayStartedShards:TRACE")
 public class IndicesShardStoreRequestIT extends ESIntegTestCase {
 
     @Override
@@ -151,7 +152,6 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
         assertThat(shardStatuses.get(index1).size(), equalTo(2));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/12416")
     public void testCorruptedShards() throws Exception {
         String index = "test";
         internalCluster().ensureAtLeastNumDataNodes(2);
@@ -175,7 +175,9 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
             for (Integer shardId : indexShards.shardIds()) {
                 IndexShard shard = indexShards.getShard(shardId);
                 if (randomBoolean()) {
+                    logger.debug("--> failing shard [{}] on node [{}]", shardId, node);
                     shard.failShard("test", new CorruptIndexException("test corrupted", ""));
+                    logger.debug("--> failed shard [{}] on node [{}]", shardId, node);
                     Set nodes = corruptedShardIDMap.get(shardId);
                     if (nodes == null) {
                         nodes = new HashSet<>();
@@ -194,9 +196,11 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
             for (IndicesShardStoresResponse.StoreStatus status : shardStatus.value) {
                 if (corruptedShardIDMap.containsKey(shardStatus.key)
                         && corruptedShardIDMap.get(shardStatus.key).contains(status.getNode().getName())) {
-                    assertThat(status.getStoreException(), notNullValue());
+                    assertThat("shard [" + shardStatus.key + "] is failed on node [" + status.getNode().getName() + "]",
+                        status.getStoreException(), notNullValue());
                 } else {
-                    assertNull(status.getStoreException());
+                    assertNull("shard [" + shardStatus.key + "] is not failed on node [" + status.getNode().getName() + "]",
+                        status.getStoreException());
                 }
             }
         }

From c6b73239ae1f3c5937d4b87055798b31c7e37b99 Mon Sep 17 00:00:00 2001
From: Mayya Sharipova 
Date: Thu, 30 Nov 2017 11:54:39 -0500
Subject: [PATCH 136/297] Limit the number of tokens produced by _analyze
 (#27529)

Add an index level setting `index.analyze.max_token_count` to control
the number of generated tokens in the  _analyze endpoint.
Defaults to 10000.

Throw an error if the number of generated tokens exceeds this limit.

Closes #27038
---
 .../analyze/TransportAnalyzeAction.java       | 50 +++++++---
 .../common/settings/IndexScopedSettings.java  |  1 +
 .../elasticsearch/index/IndexSettings.java    | 23 +++++
 .../indices/TransportAnalyzeActionTests.java  | 97 +++++++++++++++----
 docs/reference/index-modules.asciidoc         |  5 +
 docs/reference/indices/analyze.asciidoc       | 36 +++++++
 .../migration/migrate_7_0/analysis.asciidoc   |  7 ++
 .../test/indices.analyze/20_analyze_limit.yml | 52 ++++++++++
 8 files changed, 241 insertions(+), 30 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/20_analyze_limit.yml

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
index 48a956b6feb..99fe07a1f49 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
@@ -158,15 +158,18 @@ public class TransportAnalyzeAction extends TransportSingleShardAction simpleAnalyze(AnalyzeRequest request, Analyzer analyzer, String field) {
+    private static List simpleAnalyze(AnalyzeRequest request,
+            Analyzer analyzer, String field, int maxTokenCount) {
+        TokenCounter tc = new TokenCounter(maxTokenCount);
         List tokens = new ArrayList<>();
         int lastPosition = -1;
         int lastOffset = 0;
@@ -267,7 +272,7 @@ public class TransportAnalyzeAction extends TransportSingleShardAction includeAttributes = new HashSet<>();
         if (request.attributes() != null) {
@@ -307,7 +312,7 @@ public class TransportAnalyzeAction extends TransportSingleShardAction maxTokenCount) {
+                throw new IllegalStateException(
+                    "The number of tokens produced by calling _analyze has exceeded the allowed maximum of [" + maxTokenCount + "]."
+                        + " This limit can be set by changing the [index.analyze.max_token_count] index level setting.");
+            }
+        }
+    }
+
     private static class TokenListCreator {
         int lastPosition = -1;
         int lastOffset = 0;
         List tokens;
+        private TokenCounter tc;
 
-        TokenListCreator() {
+        TokenListCreator(int maxTokenCount) {
             tokens = new ArrayList<>();
+            tc = new TokenCounter(maxTokenCount);
         }
 
         private void analyze(TokenStream stream, Analyzer analyzer, String field, Set includeAttributes) {
@@ -433,7 +457,7 @@ public class TransportAnalyzeAction extends TransportSingleShardAction MAX_SCRIPT_FIELDS_SETTING =
         Setting.intSetting("index.max_script_fields", 32, 0, Property.Dynamic, Property.IndexScope);
 
+    /**
+     * A setting describing the maximum number of tokens that can be
+     * produced using _analyze API. The default maximum of 10000 is defensive
+     * to prevent generating too many token objects.
+     */
+    public static final Setting MAX_TOKEN_COUNT_SETTING =
+        Setting.intSetting("index.analyze.max_token_count", 10000, 1, Property.Dynamic, Property.IndexScope);
+
     /**
      * Index setting describing for NGramTokenizer and NGramTokenFilter
      * the maximum difference between
@@ -262,6 +270,7 @@ public final class IndexSettings {
     private volatile int maxRescoreWindow;
     private volatile int maxDocvalueFields;
     private volatile int maxScriptFields;
+    private volatile int maxTokenCount;
     private volatile int maxNgramDiff;
     private volatile int maxShingleDiff;
     private volatile boolean TTLPurgeDisabled;
@@ -369,6 +378,7 @@ public final class IndexSettings {
         maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING);
         maxDocvalueFields = scopedSettings.get(MAX_DOCVALUE_FIELDS_SEARCH_SETTING);
         maxScriptFields = scopedSettings.get(MAX_SCRIPT_FIELDS_SETTING);
+        maxTokenCount = scopedSettings.get(MAX_TOKEN_COUNT_SETTING);
         maxNgramDiff = scopedSettings.get(MAX_NGRAM_DIFF_SETTING);
         maxShingleDiff = scopedSettings.get(MAX_SHINGLE_DIFF_SETTING);
         TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING);
@@ -403,6 +413,7 @@ public final class IndexSettings {
         scopedSettings.addSettingsUpdateConsumer(MAX_RESCORE_WINDOW_SETTING, this::setMaxRescoreWindow);
         scopedSettings.addSettingsUpdateConsumer(MAX_DOCVALUE_FIELDS_SEARCH_SETTING, this::setMaxDocvalueFields);
         scopedSettings.addSettingsUpdateConsumer(MAX_SCRIPT_FIELDS_SETTING, this::setMaxScriptFields);
+        scopedSettings.addSettingsUpdateConsumer(MAX_TOKEN_COUNT_SETTING, this::setMaxTokenCount);
         scopedSettings.addSettingsUpdateConsumer(MAX_NGRAM_DIFF_SETTING, this::setMaxNgramDiff);
         scopedSettings.addSettingsUpdateConsumer(MAX_SHINGLE_DIFF_SETTING, this::setMaxShingleDiff);
         scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer);
@@ -676,6 +687,18 @@ public final class IndexSettings {
         this.maxDocvalueFields = maxDocvalueFields;
     }
 
+    /**
+     * Returns the maximum number of tokens that can be produced
+     */
+    public int getMaxTokenCount() {
+        return maxTokenCount;
+    }
+
+    private void setMaxTokenCount(int maxTokenCount) {
+        this.maxTokenCount = maxTokenCount;
+    }
+
+
     /**
      * Returns the maximum allowed difference between max and min length of ngram
      */
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
index 90857da0be0..515606dd10b 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
@@ -61,6 +61,8 @@ public class TransportAnalyzeActionTests extends ESTestCase {
     private IndexAnalyzers indexAnalyzers;
     private AnalysisRegistry registry;
     private Environment environment;
+    private int maxTokenCount;
+    private int idxMaxTokenCount;
 
     @Override
     public void setUp() throws Exception {
@@ -73,6 +75,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
                 .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard")
                 .put("index.analysis.analyzer.custom_analyzer.filter", "mock")
                 .put("index.analysis.normalizer.my_normalizer.type", "custom")
+                .put("index.analyze.max_token_count", 100)
                 .putList("index.analysis.normalizer.my_normalizer.filter", "lowercase").build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
         environment = TestEnvironment.newEnvironment(settings);
@@ -116,6 +119,8 @@ public class TransportAnalyzeActionTests extends ESTestCase {
         };
         registry = new AnalysisModule(environment, singletonList(plugin)).getAnalysisRegistry();
         indexAnalyzers = registry.build(idxSettings);
+        maxTokenCount = IndexSettings.MAX_TOKEN_COUNT_SETTING.getDefault(settings);
+        idxMaxTokenCount = idxSettings.getMaxTokenCount();
     }
 
     /**
@@ -126,7 +131,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
         AnalyzeRequest request = new AnalyzeRequest();
         request.text("the quick brown fox");
         request.analyzer("standard");
-        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, null, registry, environment);
+        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, null, registry, environment, maxTokenCount);
         List tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
 
@@ -135,7 +140,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
         request.text("the qu1ck brown fox");
         request.tokenizer("standard");
         request.addTokenFilter("mock");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, randomBoolean() ? indexAnalyzers : null, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, "text", null, randomBoolean() ? indexAnalyzers : null, registry, environment, maxTokenCount);
         tokens = analyze.getTokens();
         assertEquals(3, tokens.size());
         assertEquals("qu1ck", tokens.get(0).getTerm());
@@ -147,7 +152,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
         request.text("the qu1ck brown fox");
         request.tokenizer("standard");
         request.addCharFilter("append_foo");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, randomBoolean() ? indexAnalyzers : null, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, "text", null, randomBoolean() ? indexAnalyzers : null, registry, environment, maxTokenCount);
         tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
@@ -161,7 +166,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
         request.tokenizer("standard");
         request.addCharFilter("append");
         request.text("the qu1ck brown fox");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, randomBoolean() ? indexAnalyzers : null, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, "text", null, randomBoolean() ? indexAnalyzers : null, registry, environment, maxTokenCount);
         tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
@@ -174,7 +179,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
         AnalyzeRequest request = new AnalyzeRequest();
         request.analyzer("standard");
         request.text("the 1 brown fox");
-        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, null, registry, environment);
+        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, null, registry, environment, maxTokenCount);
         List tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
@@ -206,7 +211,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
         AnalyzeRequest request = new AnalyzeRequest();
         request.text("the quick brown fox");
         request.analyzer("custom_analyzer");
-        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment);
+        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment, maxTokenCount);
         List tokens = analyze.getTokens();
         assertEquals(3, tokens.size());
         assertEquals("quick", tokens.get(0).getTerm());
@@ -214,7 +219,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
         assertEquals("fox", tokens.get(2).getTerm());
 
         request.analyzer("standard");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment, maxTokenCount);
         tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
@@ -225,7 +230,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
         // Switch the analyzer out for just a tokenizer
         request.analyzer(null);
         request.tokenizer("standard");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment, maxTokenCount);
         tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
@@ -235,7 +240,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
 
         // Now try applying our token filter
         request.addTokenFilter("mock");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment);
+        analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment, maxTokenCount);
         tokens = analyze.getTokens();
         assertEquals(3, tokens.size());
         assertEquals("quick", tokens.get(0).getTerm());
@@ -249,7 +254,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
                 new AnalyzeRequest()
                     .analyzer("custom_analyzer")
                     .text("the qu1ck brown fox-dog"),
-                "text", null, null, registry, environment));
+                "text", null, null, registry, environment, maxTokenCount));
         assertEquals(e.getMessage(), "failed to find global analyzer [custom_analyzer]");
     }
 
@@ -260,7 +265,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
                 new AnalyzeRequest()
                     .analyzer("foobar")
                     .text("the qu1ck brown fox"),
-                "text", null, notGlobal ? indexAnalyzers : null, registry, environment));
+                "text", null, notGlobal ? indexAnalyzers : null, registry, environment, maxTokenCount));
         if (notGlobal) {
             assertEquals(e.getMessage(), "failed to find analyzer [foobar]");
         } else {
@@ -272,7 +277,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
                 new AnalyzeRequest()
                     .tokenizer("foobar")
                     .text("the qu1ck brown fox"),
-                "text", null, notGlobal ? indexAnalyzers : null, registry, environment));
+                "text", null, notGlobal ? indexAnalyzers : null, registry, environment, maxTokenCount));
         if (notGlobal) {
             assertEquals(e.getMessage(), "failed to find tokenizer under [foobar]");
         } else {
@@ -285,7 +290,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
                     .tokenizer("whitespace")
                     .addTokenFilter("foobar")
                     .text("the qu1ck brown fox"),
-                "text", null, notGlobal ? indexAnalyzers : null, registry, environment));
+                "text", null, notGlobal ? indexAnalyzers : null, registry, environment, maxTokenCount));
         if (notGlobal) {
             assertEquals(e.getMessage(), "failed to find token filter under [foobar]");
         } else {
@@ -299,7 +304,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
                     .addTokenFilter("lowercase")
                     .addCharFilter("foobar")
                     .text("the qu1ck brown fox"),
-                "text", null, notGlobal ? indexAnalyzers : null, registry, environment));
+                "text", null, notGlobal ? indexAnalyzers : null, registry, environment, maxTokenCount));
         if (notGlobal) {
             assertEquals(e.getMessage(), "failed to find char filter under [foobar]");
         } else {
@@ -311,7 +316,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
                 new AnalyzeRequest()
                     .normalizer("foobar")
                     .text("the qu1ck brown fox"),
-                "text", null, indexAnalyzers, registry, environment));
+                "text", null, indexAnalyzers, registry, environment, maxTokenCount));
         assertEquals(e.getMessage(), "failed to find normalizer under [foobar]");
     }
 
@@ -320,7 +325,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
         request.tokenizer("whitespace");
         request.addTokenFilter("stop"); // stop token filter is not prebuilt in AnalysisModule#setupPreConfiguredTokenFilters()
         request.text("the quick brown fox");
-        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment);
+        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment, maxTokenCount);
         List tokens = analyze.getTokens();
         assertEquals(3, tokens.size());
         assertEquals("quick", tokens.get(0).getTerm());
@@ -332,10 +337,68 @@ public class TransportAnalyzeActionTests extends ESTestCase {
         AnalyzeRequest request = new AnalyzeRequest("index");
         request.normalizer("my_normalizer");
         request.text("ABc");
-        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment);
+        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment, maxTokenCount);
         List tokens = analyze.getTokens();
 
         assertEquals(1, tokens.size());
         assertEquals("abc", tokens.get(0).getTerm());
     }
+
+    /**
+     * This test is equivalent of calling _analyze without a specific index.
+     * The default value for the maximum token count is used.
+     */
+    public void testExceedDefaultMaxTokenLimit() throws IOException{
+        // create a string with No. words more than maxTokenCount
+        StringBuilder sbText = new StringBuilder();
+        for (int i = 0; i <= maxTokenCount; i++){
+            sbText.append('a');
+            sbText.append(' ');
+        }
+        String text = sbText.toString();
+
+        // request with explain=false to test simpleAnalyze path in TransportAnalyzeAction
+        AnalyzeRequest request = new AnalyzeRequest();
+        request.text(text);
+        request.analyzer("standard");
+        IllegalStateException e = expectThrows(IllegalStateException.class,
+            () -> TransportAnalyzeAction.analyze(
+                request, "text", null, null, registry, environment, maxTokenCount));
+        assertEquals(e.getMessage(), "The number of tokens produced by calling _analyze has exceeded the allowed maximum of ["
+            + maxTokenCount + "]." + " This limit can be set by changing the [index.analyze.max_token_count] index level setting.");
+
+        // request with explain=true to test detailAnalyze path in TransportAnalyzeAction
+        AnalyzeRequest request2 = new AnalyzeRequest();
+        request2.text(text);
+        request2.analyzer("standard");
+        request2.explain(true);
+        IllegalStateException e2 = expectThrows(IllegalStateException.class,
+            () -> TransportAnalyzeAction.analyze(
+                request2, "text", null, null, registry, environment, maxTokenCount));
+        assertEquals(e2.getMessage(), "The number of tokens produced by calling _analyze has exceeded the allowed maximum of ["
+            + maxTokenCount + "]." + " This limit can be set by changing the [index.analyze.max_token_count] index level setting.");
+    }
+
+    /**
+     * This test is equivalent of calling _analyze against a specific index.
+     * The index specific value for the maximum token count is used.
+     */
+    public void testExceedSetMaxTokenLimit() throws IOException{
+        // create a string with No. words more than idxMaxTokenCount
+        StringBuilder sbText = new StringBuilder();
+        for (int i = 0; i <= idxMaxTokenCount; i++){
+            sbText.append('a');
+            sbText.append(' ');
+        }
+        String text = sbText.toString();
+
+        AnalyzeRequest request = new AnalyzeRequest();
+        request.text(text);
+        request.analyzer("standard");
+        IllegalStateException e = expectThrows(IllegalStateException.class,
+            () -> TransportAnalyzeAction.analyze(
+                request, "text", null, indexAnalyzers, registry, environment, idxMaxTokenCount));
+        assertEquals(e.getMessage(), "The number of tokens produced by calling _analyze has exceeded the allowed maximum of ["
+            + idxMaxTokenCount + "]." + " This limit can be set by changing the [index.analyze.max_token_count] index level setting.");
+    }
 }
diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index 999bd948a15..70f3abe64c3 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -193,6 +193,11 @@ specific index module:
     Maximum number of refresh listeners available on each shard of the index.
     These listeners are used to implement <>.
 
+ `index.analyze.max_token_count`::
+
+    The maximum number of tokens that can be produced using _analyze API.
+    Defaults to `10000`.
+
 
 [float]
 === Settings in other index modules
diff --git a/docs/reference/indices/analyze.asciidoc b/docs/reference/indices/analyze.asciidoc
index 904fa437913..0398e359625 100644
--- a/docs/reference/indices/analyze.asciidoc
+++ b/docs/reference/indices/analyze.asciidoc
@@ -207,3 +207,39 @@ The request returns the following result:
 --------------------------------------------------
 // TESTRESPONSE
 <1> Output only "keyword" attribute, since specify "attributes" in the request.
+
+[[tokens-limit-settings]]
+[float]
+== Settings to prevent tokens explosion
+Generating excessive amount of tokens may cause a node to run out of memory.
+The following setting allows to limit the number of tokens that can be produced:
+
+`index.analyze.max_token_count`::
+    The maximum number of tokens that can be produced using `_analyze` API.
+    The default value is `10000`. If more than this limit of tokens gets
+    generated, an error will be thrown. The `_analyze` endpoint without a specified
+    index will always use `10000` value as a limit. This setting allows you to control
+    the limit for a specific index:
+
+
+[source,js]
+--------------------------------------------------
+PUT analyze_sample
+{
+  "settings" : {
+    "index.analyze.max_token_count" : 20000
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+
+[source,js]
+--------------------------------------------------
+GET analyze_sample/_analyze
+{
+  "text" : "this is a test"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:analyze_sample]
\ No newline at end of file
diff --git a/docs/reference/migration/migrate_7_0/analysis.asciidoc b/docs/reference/migration/migrate_7_0/analysis.asciidoc
index 9a1b0aecd59..ebe2cf1dfc2 100644
--- a/docs/reference/migration/migrate_7_0/analysis.asciidoc
+++ b/docs/reference/migration/migrate_7_0/analysis.asciidoc
@@ -6,3 +6,10 @@
 The `delimited_payload_filter` is renamed to `delimited_payload`, the old name is 
 deprecated and will be removed at some point, so it should be replaced by 
 `delimited_payload`.
+
+
+==== Limiting the number of tokens produced by _analyze
+
+To safeguard against out of memory errors, the number of tokens that can be produced
+using the `_analyze` endpoint has been limited to 10000. This default limit can be changed
+for a particular index with the index setting `index.analyze.max_token_count`.
\ No newline at end of file
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/20_analyze_limit.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/20_analyze_limit.yml
new file mode 100644
index 00000000000..87d3b77aee3
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/20_analyze_limit.yml
@@ -0,0 +1,52 @@
+---
+setup:
+  - do:
+      indices.create:
+          index: test_1
+          body:
+              settings:
+                  index.analyze.max_token_count: 3
+
+---
+"_analyze with No. generated tokens less than or equal to index.analyze.max_token_count should succeed":
+  - skip:
+      version: " - 6.99.99"
+      reason:  index.analyze.max_token_count setting has been added in 7.0.0
+  - do:
+      indices.analyze:
+          index:  test_1
+          body:
+              text: This should succeed
+              analyzer: standard
+  - length: { tokens: 3 }
+  - match:  { tokens.0.token: this }
+  - match:  { tokens.1.token: should }
+  - match:  { tokens.2.token: succeed }
+
+---
+"_analyze with No. generated tokens more than index.analyze.max_token_count should fail":
+  - skip:
+      version: " - 6.99.99"
+      reason:  index.analyze.max_token_count setting has been added in 7.0.0
+  - do:
+      catch: /The number of tokens produced by calling _analyze has exceeded the allowed maximum of \[3\]. This limit can be set by changing the \[index.analyze.max_token_count\] index level setting\./
+      indices.analyze:
+          index:  test_1
+          body:
+              text: This should fail as it exceeds limit
+              analyzer: standard
+
+
+---
+"_analyze with explain with No. generated tokens more than index.analyze.max_token_count should fail":
+  - skip:
+      version: " - 6.99.99"
+      reason:  index.analyze.max_token_count setting has been added in 7.0.0
+  - do:
+      catch: /The number of tokens produced by calling _analyze has exceeded the allowed maximum of \[3\]. This limit can be set by changing the \[index.analyze.max_token_count\] index level setting\./
+      indices.analyze:
+          index:  test_1
+          body:
+              text: This should fail as it exceeds limit
+              analyzer: standard
+              explain: true

From b1162215408a7a11bc37ddb60a371ebc28e7e25d Mon Sep 17 00:00:00 2001
From: Simon Willnauer 
Date: Thu, 30 Nov 2017 19:04:05 +0100
Subject: [PATCH 137/297] Ensure shard is refreshed once it's inactive (#27559)

Once a shard goes inactive we want the shard to be refreshed if
the refresh interval is default since we might hold on to unnecessary
segments and in the inactive case we stopped indexing and can release
old segments.

Relates to #27500
---
 .../java/org/elasticsearch/index/shard/IndexShard.java   | 8 +++++---
 .../org/elasticsearch/index/shard/IndexShardTests.java   | 9 +++++++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 7b29b2b632e..0f554b6ace1 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -873,7 +873,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         long numDeletedDocs = 0;
         long sizeInBytes = 0;
         try (Engine.Searcher searcher = acquireSearcher("docStats", Engine.SearcherScope.INTERNAL)) {
-            // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accesssed only which will cause
+            // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause
             // the next scheduled refresh to go through and refresh the stats as well
             markSearcherAccessed();
             for (LeafReaderContext reader : searcher.reader().leaves()) {
@@ -972,7 +972,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     public CompletionStats completionStats(String... fields) {
         CompletionStats completionStats = new CompletionStats();
         try (Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) {
-            // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accesssed only which will cause
+            // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause
             // the next scheduled refresh to go through and refresh the stats as well
             markSearcherAccessed();
             completionStats.add(CompletionFieldStats.completionStats(currentSearcher.reader(), fields));
@@ -2457,7 +2457,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         boolean listenerNeedsRefresh = refreshListeners.refreshNeeded();
         if (isReadAllowed() && (listenerNeedsRefresh || getEngine().refreshNeeded())) {
             if (listenerNeedsRefresh == false // if we have a listener that is waiting for a refresh we need to force it
-                && isSearchIdle() && indexSettings.isExplicitRefresh() == false) {
+                && isSearchIdle()
+                && indexSettings.isExplicitRefresh() == false
+                && active.get()) { // it must be active otherwise we might not free up segment memory once the shard became inactive
                 // lets skip this refresh since we are search idle and
                 // don't necessarily need to refresh. the next searcher access will register a refreshListener and that will
                 // cause the next schedule to refresh.
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 00f29dc252d..b3a0f4b88de 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -2685,6 +2685,15 @@ public class IndexShardTests extends IndexShardTestCase {
 
         });
         latch1.await();
+
+        indexDoc(primary, "test", "2", "{\"foo\" : \"bar\"}");
+        assertFalse(primary.scheduledRefresh());
+        assertTrue(primary.isSearchIdle());
+        primary.checkIdle(0);
+        assertTrue(primary.scheduledRefresh()); // make sure we refresh once the shard is inactive
+        try (Engine.Searcher searcher = primary.acquireSearcher("test")) {
+            assertEquals(3, searcher.reader().numDocs());
+        }
         closeShards(primary);
     }
 

From d30c88789312baec7a5c83e4043580306ea5cf61 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Thu, 30 Nov 2017 14:08:33 -0500
Subject: [PATCH 138/297] Use private directory for temporary files

This change ensures that the temporary directory used for java.io.tmpdir
is a private temporary directory. To achieve this we use mktemp on macOS
and Linux to give us a private temporary directory and the value of the
environment variable TMP on Windows. For this to work with our
packaging, we add java.io.tmpdir=${ES_TMPDIR} to our packaged
jvm.options, we set ES_TMPDIR respectively in our startup scripts, and
resolve the value of the template ${ES_TMPDIR} at startup.

Relates #27609
---
 distribution/src/main/resources/bin/elasticsearch         | 3 ++-
 distribution/src/main/resources/bin/elasticsearch-env     | 8 ++++++++
 distribution/src/main/resources/bin/elasticsearch-env.bat | 4 ++++
 .../src/main/resources/bin/elasticsearch-service.bat      | 2 +-
 distribution/src/main/resources/bin/elasticsearch.bat     | 2 +-
 distribution/src/main/resources/config/jvm.options        | 2 ++
 6 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch b/distribution/src/main/resources/bin/elasticsearch
index 6602a9aa7ee..6a63d2e9aa8 100755
--- a/distribution/src/main/resources/bin/elasticsearch
+++ b/distribution/src/main/resources/bin/elasticsearch
@@ -24,7 +24,8 @@ parse_jvm_options() {
 
 ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options
 
-ES_JAVA_OPTS="`parse_jvm_options "$ES_JVM_OPTIONS"` $ES_JAVA_OPTS"
+JVM_OPTIONS=`parse_jvm_options "$ES_JVM_OPTIONS"`
+ES_JAVA_OPTS="${JVM_OPTIONS//\$\{ES_TMPDIR\}/$ES_TMPDIR} $ES_JAVA_OPTS"
 
 # manual parsing to find out, if process should be detached
 if ! echo $* | grep -E '(^-d |-d$| -d |--daemonize$|--daemonize )' > /dev/null; then
diff --git a/distribution/src/main/resources/bin/elasticsearch-env b/distribution/src/main/resources/bin/elasticsearch-env
index 7e74195e607..2b376bd47b3 100644
--- a/distribution/src/main/resources/bin/elasticsearch-env
+++ b/distribution/src/main/resources/bin/elasticsearch-env
@@ -73,3 +73,11 @@ if [ -z "$ES_PATH_CONF" ]; then
   echo "ES_PATH_CONF must be set to the configuration path"
   exit 1
 fi
+
+if [ -z "$ES_TMPDIR" ]; then
+  if [ "`uname`" == "Darwin" ]; then
+    ES_TMPDIR=`mktemp -d -t elasticsearch`
+  else
+    ES_TMPDIR=`mktemp -d -t elasticsearch.XXXXXXXX`
+  fi
+fi
diff --git a/distribution/src/main/resources/bin/elasticsearch-env.bat b/distribution/src/main/resources/bin/elasticsearch-env.bat
index 00cc7fa0980..e80a8ce258e 100644
--- a/distribution/src/main/resources/bin/elasticsearch-env.bat
+++ b/distribution/src/main/resources/bin/elasticsearch-env.bat
@@ -49,3 +49,7 @@ set HOSTNAME=%COMPUTERNAME%
 if not defined ES_PATH_CONF (
   set ES_PATH_CONF=!ES_HOME!\config
 )
+
+if not defined ES_TMPDIR (
+  set ES_TMPDIR=!TMP!\elasticsearch
+)
diff --git a/distribution/src/main/resources/bin/elasticsearch-service.bat b/distribution/src/main/resources/bin/elasticsearch-service.bat
index dfb854a4708..e8be9485349 100644
--- a/distribution/src/main/resources/bin/elasticsearch-service.bat
+++ b/distribution/src/main/resources/bin/elasticsearch-service.bat
@@ -104,7 +104,7 @@ if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
 
 @setlocal
 for /F "usebackq delims=" %%a in (`findstr /b \- "%ES_JVM_OPTIONS%" ^| findstr /b /v "\-server \-client"`) do set JVM_OPTIONS=!JVM_OPTIONS!%%a;
-@endlocal & set ES_JAVA_OPTS=%JVM_OPTIONS%%ES_JAVA_OPTS%
+@endlocal & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!%%ES_JAVA_OPTS%
 
 if "%ES_JAVA_OPTS:~-1%"==";" set ES_JAVA_OPTS=%ES_JAVA_OPTS:~0,-1%
 
diff --git a/distribution/src/main/resources/bin/elasticsearch.bat b/distribution/src/main/resources/bin/elasticsearch.bat
index 51f20476790..5c172d20b82 100644
--- a/distribution/src/main/resources/bin/elasticsearch.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.bat
@@ -47,7 +47,7 @@ set "ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options"
 rem extract the options from the JVM options file %ES_JVM_OPTIONS%
 rem such options are the lines beginning with '-', thus "findstr /b"
 for /F "usebackq delims=" %%a in (`findstr /b \- "%ES_JVM_OPTIONS%"`) do set JVM_OPTIONS=!JVM_OPTIONS! %%a
-@endlocal & set ES_JAVA_OPTS=%JVM_OPTIONS% %ES_JAVA_OPTS%
+@endlocal & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!% %ES_JAVA_OPTS%
 
 %JAVA% %ES_JAVA_OPTS% -Delasticsearch -Des.path.home="%ES_HOME%" -Des.path.conf="%ES_PATH_CONF%" -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams!
 
diff --git a/distribution/src/main/resources/config/jvm.options b/distribution/src/main/resources/config/jvm.options
index 52bd4244bb8..f5d527eec87 100644
--- a/distribution/src/main/resources/config/jvm.options
+++ b/distribution/src/main/resources/config/jvm.options
@@ -72,6 +72,8 @@
 -Dlog4j.shutdownHookEnabled=false
 -Dlog4j2.disable.jmx=true
 
+-Djava.io.tmpdir=${ES_TMPDIR}
+
 ## heap dumps
 
 # generate a heap dump when an allocation from the Java heap fails

From 67cd1e9c5f8957c28c954a407fd0f9a3401c869e Mon Sep 17 00:00:00 2001
From: Simon Willnauer 
Date: Thu, 30 Nov 2017 20:44:05 +0100
Subject: [PATCH 139/297] Reset LiveVersionMap on sync commit (#27534)

Today we carry on the size of the live version map to ensure that
we minimze rehashing. Yet, once we are idle or we can issue a sync-commit
we can resize it to defaults to free up memory.

Relates to #27516
---
 .../index/engine/DeleteVersionValue.java      |  18 ++
 .../index/engine/InternalEngine.java          |  10 +-
 .../index/engine/LiveVersionMap.java          |  29 +++-
 .../index/engine/VersionValue.java            |  21 +++
 .../index/engine/LiveVersionMapTests.java     | 160 ++++++++++++++++++
 5 files changed, 227 insertions(+), 11 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java b/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java
index 899c06eb196..d2b2e24c616 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java
@@ -44,6 +44,24 @@ class DeleteVersionValue extends VersionValue {
         return BASE_RAM_BYTES_USED;
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+
+        DeleteVersionValue that = (DeleteVersionValue) o;
+
+        return time == that.time;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + (int) (time ^ (time >>> 32));
+        return result;
+    }
+
     @Override
     public String toString() {
         return "DeleteVersionValue{" +
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 6449c979de4..82461ca4a99 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -560,7 +560,7 @@ public class InternalEngine extends Engine {
             ensureOpen();
             SearcherScope scope;
             if (get.realtime()) {
-                VersionValue versionValue = versionMap.getUnderLock(get.uid());
+                VersionValue versionValue = versionMap.getUnderLock(get.uid().bytes());
                 if (versionValue != null) {
                     if (versionValue.isDelete()) {
                         return GetResult.NOT_EXISTS;
@@ -598,7 +598,7 @@ public class InternalEngine extends Engine {
     private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnSeqNo(final Operation op) throws IOException {
         assert op.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "resolving ops based on seq# but no seqNo is found";
         final OpVsLuceneDocStatus status;
-        final VersionValue versionValue = versionMap.getUnderLock(op.uid());
+        final VersionValue versionValue = versionMap.getUnderLock(op.uid().bytes());
         assert incrementVersionLookup();
         if (versionValue != null) {
             if  (op.seqNo() > versionValue.seqNo ||
@@ -635,7 +635,7 @@ public class InternalEngine extends Engine {
     /** resolves the current version of the document, returning null if not found */
     private VersionValue resolveDocVersion(final Operation op) throws IOException {
         assert incrementVersionLookup(); // used for asserting in tests
-        VersionValue versionValue = versionMap.getUnderLock(op.uid());
+        VersionValue versionValue = versionMap.getUnderLock(op.uid().bytes());
         if (versionValue == null) {
             assert incrementIndexVersionLookup(); // used for asserting in tests
             final long currentVersion = loadCurrentVersionFromIndex(op.uid());
@@ -1048,7 +1048,7 @@ public class InternalEngine extends Engine {
      * Asserts that the doc in the index operation really doesn't exist
      */
     private boolean assertDocDoesNotExist(final Index index, final boolean allowDeleted) throws IOException {
-        final VersionValue versionValue = versionMap.getUnderLock(index.uid());
+        final VersionValue versionValue = versionMap.getUnderLock(index.uid().bytes());
         if (versionValue != null) {
             if (versionValue.isDelete() == false || allowDeleted == false) {
                 throw new AssertionError("doc [" + index.type() + "][" + index.id() + "] exists in version map (version " + versionValue + ")");
@@ -1376,6 +1376,8 @@ public class InternalEngine extends Engine {
             commitIndexWriter(indexWriter, translog, syncId);
             logger.debug("successfully sync committed. sync id [{}].", syncId);
             lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
+            // we are guaranteed to have no operations in the version map here!
+            versionMap.adjustMapSizeUnderLock();
             return SyncedFlushResult.SUCCESS;
         } catch (IOException ex) {
             maybeFailEngine("sync commit", ex);
diff --git a/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java b/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
index aef41d9d162..48d57ee7eec 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.engine;
 
-import org.apache.lucene.index.Term;
 import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
@@ -35,6 +34,18 @@ import java.util.concurrent.atomic.AtomicLong;
 /** Maps _uid value to its version information. */
 class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
 
+    /**
+     * Resets the internal map and adjusts it's capacity as if there were no indexing operations.
+     * This must be called under write lock in the engine
+     */
+    void adjustMapSizeUnderLock() {
+        if (maps.current.isEmpty() == false || maps.old.isEmpty() == false) {
+            assert false : "map must be empty"; // fail hard if not empty and fail with assertion in tests to ensure we never swallow it
+            throw new IllegalStateException("map must be empty");
+        }
+        maps = new Maps();
+    }
+
     private static class Maps {
 
         // All writes (adds and deletes) go into here:
@@ -50,7 +61,7 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
 
         Maps() {
             this(ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency(),
-                 ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency());
+                 Collections.emptyMap());
         }
     }
 
@@ -121,21 +132,21 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
     }
 
     /** Returns the live version (add or delete) for this uid. */
-    VersionValue getUnderLock(final Term uid) {
+    VersionValue getUnderLock(final BytesRef uid) {
         Maps currentMaps = maps;
 
         // First try to get the "live" value:
-        VersionValue value = currentMaps.current.get(uid.bytes());
+        VersionValue value = currentMaps.current.get(uid);
         if (value != null) {
             return value;
         }
 
-        value = currentMaps.old.get(uid.bytes());
+        value = currentMaps.old.get(uid);
         if (value != null) {
             return value;
         }
 
-        return tombstones.get(uid.bytes());
+        return tombstones.get(uid);
     }
 
     /** Adds this uid/version to the pending adds map. */
@@ -250,4 +261,8 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
         // TODO: useful to break down RAM usage here?
         return Collections.emptyList();
     }
-}
+
+    /** Returns the current internal versions as a point in time snapshot*/
+    Map getAllCurrent() {
+        return maps.current;
+    }}
diff --git a/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java b/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
index f3d9618838f..e2a2614d6c1 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
@@ -57,10 +57,31 @@ class VersionValue implements Accountable {
         return Collections.emptyList();
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        VersionValue that = (VersionValue) o;
+
+        if (version != that.version) return false;
+        if (seqNo != that.seqNo) return false;
+        return term == that.term;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = (int) (version ^ (version >>> 32));
+        result = 31 * result + (int) (seqNo ^ (seqNo >>> 32));
+        result = 31 * result + (int) (term ^ (term >>> 32));
+        return result;
+    }
+
     @Override
     public String toString() {
         return "VersionValue{" +
             "version=" + version +
+
             ", seqNo=" + seqNo +
             ", term=" + term +
             '}';
diff --git a/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java b/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
index 97799f8c46a..77e5b55ac57 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
@@ -19,12 +19,25 @@
 
 package org.elasticsearch.index.engine;
 
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.RamUsageTester;
 import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.Assertions;
 import org.elasticsearch.bootstrap.JavaVersion;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.util.concurrent.KeyedLock;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+
 public class LiveVersionMapTests extends ESTestCase {
 
     public void testRamBytesUsed() throws Exception {
@@ -57,4 +70,151 @@ public class LiveVersionMapTests extends ESTestCase {
         assertEquals(actualRamBytesUsed, estimatedRamBytesUsed, actualRamBytesUsed / 4);
     }
 
+    private BytesRef uid(String string) {
+        BytesRefBuilder builder = new BytesRefBuilder();
+        builder.copyChars(string);
+        // length of the array must be the same as the len of the ref... there is an assertion in LiveVersionMap#putUnderLock
+        return BytesRef.deepCopyOf(builder.get());
+    }
+
+    public void testBasics() throws IOException {
+        LiveVersionMap map = new LiveVersionMap();
+        map.putUnderLock(uid("test"), new VersionValue(1,1,1));
+        assertEquals(new VersionValue(1,1,1), map.getUnderLock(uid("test")));
+        map.beforeRefresh();
+        assertEquals(new VersionValue(1,1,1), map.getUnderLock(uid("test")));
+        map.afterRefresh(randomBoolean());
+        assertNull(map.getUnderLock(uid("test")));
+
+
+        map.putUnderLock(uid("test"), new DeleteVersionValue(1,1,1, Long.MAX_VALUE));
+        assertEquals(new DeleteVersionValue(1,1,1, Long.MAX_VALUE), map.getUnderLock(uid("test")));
+        map.beforeRefresh();
+        assertEquals(new DeleteVersionValue(1,1,1, Long.MAX_VALUE), map.getUnderLock(uid("test")));
+        map.afterRefresh(randomBoolean());
+        assertEquals(new DeleteVersionValue(1,1,1, Long.MAX_VALUE), map.getUnderLock(uid("test")));
+        map.removeTombstoneUnderLock(uid("test"));
+        assertNull(map.getUnderLock(uid("test")));
+    }
+
+
+    public void testAdjustMapSizeUnderLock() throws IOException {
+        LiveVersionMap map = new LiveVersionMap();
+        map.putUnderLock(uid("test"), new VersionValue(1,1,1));
+        boolean withinRefresh = randomBoolean();
+        if (withinRefresh) {
+            map.beforeRefresh();
+        }
+        assertEquals(new VersionValue(1,1,1), map.getUnderLock(uid("test")));
+        final String msg;
+        if (Assertions.ENABLED) {
+            msg = expectThrows(AssertionError.class, map::adjustMapSizeUnderLock).getMessage();
+        } else {
+            msg = expectThrows(IllegalStateException.class, map::adjustMapSizeUnderLock).getMessage();
+        }
+        assertEquals("map must be empty", msg);
+        assertEquals(new VersionValue(1,1,1), map.getUnderLock(uid("test")));
+        if (withinRefresh == false) {
+            map.beforeRefresh();
+        }
+        map.afterRefresh(randomBoolean());
+        Map allCurrent = map.getAllCurrent();
+        map.adjustMapSizeUnderLock();
+        assertNotSame(allCurrent, map.getAllCurrent());
+    }
+
+    public void testConcurrently() throws IOException, InterruptedException {
+        HashSet keySet = new HashSet<>();
+        int numKeys = randomIntBetween(50, 200);
+        for (int i = 0; i < numKeys; i++) {
+            keySet.add(uid(TestUtil.randomSimpleString(random(), 10, 20)));
+        }
+        List keyList = new ArrayList<>(keySet);
+        ConcurrentHashMap values = new ConcurrentHashMap<>();
+        KeyedLock keyedLock = new KeyedLock<>();
+        LiveVersionMap map = new LiveVersionMap();
+        int numThreads = randomIntBetween(2, 5);
+
+        Thread[] threads = new Thread[numThreads];
+        CountDownLatch startGun = new CountDownLatch(numThreads);
+        CountDownLatch done = new CountDownLatch(numThreads);
+        int randomValuesPerThread = randomIntBetween(5000, 20000);
+        for (int j = 0; j < threads.length; j++) {
+            threads[j] = new Thread(() -> {
+                startGun.countDown();
+                try {
+                    startGun.await();
+                } catch (InterruptedException e) {
+                    done.countDown();
+                    throw new AssertionError(e);
+                }
+                try {
+                    for (int i = 0; i < randomValuesPerThread; ++i) {
+                        BytesRef bytesRef = randomFrom(random(), keyList);
+                        try (Releasable r = keyedLock.acquire(bytesRef)) {
+                            VersionValue versionValue = values.computeIfAbsent(bytesRef,
+                                v -> new VersionValue(randomLong(), randomLong(), randomLong()));
+                            boolean isDelete = versionValue instanceof DeleteVersionValue;
+                            if (isDelete) {
+                                map.removeTombstoneUnderLock(bytesRef);
+                            }
+                            if (isDelete == false && rarely()) {
+                                versionValue = new DeleteVersionValue(versionValue.version + 1, versionValue.seqNo + 1,
+                                    versionValue.term, Long.MAX_VALUE);
+                            } else {
+                                versionValue = new VersionValue(versionValue.version + 1, versionValue.seqNo + 1, versionValue.term);
+                            }
+                            values.put(bytesRef, versionValue);
+                            map.putUnderLock(bytesRef, versionValue);
+                        }
+                    }
+                } finally {
+                    done.countDown();
+                }
+            });
+            threads[j].start();
+
+
+        }
+        do {
+            Map valueMap = new HashMap<>(map.getAllCurrent());
+            map.beforeRefresh();
+            valueMap.forEach((k, v) -> {
+                VersionValue actualValue = map.getUnderLock(k);
+                assertNotNull(actualValue);
+                assertTrue(v.version <= actualValue.version);
+            });
+            map.afterRefresh(randomBoolean());
+            valueMap.forEach((k, v) -> {
+                VersionValue actualValue = map.getUnderLock(k);
+                if (actualValue != null) {
+                    if (actualValue instanceof DeleteVersionValue) {
+                        assertTrue(v.version <= actualValue.version); // deletes can be the same version
+                    } else {
+                        assertTrue(v.version < actualValue.version);
+                    }
+
+                }
+            });
+            if (randomBoolean()) {
+                Thread.yield();
+            }
+        } while (done.getCount() != 0);
+
+        for (int j = 0; j < threads.length; j++) {
+            threads[j].join();
+        }
+        map.getAllCurrent().forEach((k, v) -> {
+            VersionValue versionValue = values.get(k);
+            assertNotNull(versionValue);
+            assertEquals(v, versionValue);
+        });
+
+        map.getAllTombstones().forEach(e -> {
+            VersionValue versionValue = values.get(e.getKey());
+            assertNotNull(versionValue);
+            assertEquals(e.getValue(), versionValue);
+            assertTrue(versionValue instanceof DeleteVersionValue);
+        });
+    }
 }

From 1e6bd99248921a5b26e5a5612c15bf60d505a693 Mon Sep 17 00:00:00 2001
From: David Turner 
Date: Thu, 30 Nov 2017 21:09:01 +0000
Subject: [PATCH 140/297] Reinstate AwaitsFix

---
 .../action/admin/indices/shards/IndicesShardStoreRequestIT.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
index ab66694cd96..be8a7894387 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
@@ -152,6 +152,7 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
         assertThat(shardStatuses.get(index1).size(), equalTo(2));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/12416")
     public void testCorruptedShards() throws Exception {
         String index = "test";
         internalCluster().ensureAtLeastNumDataNodes(2);

From 95bcee56c42dc457c4157e51b7277d99d43a666a Mon Sep 17 00:00:00 2001
From: Denis Gladkikh 
Date: Thu, 30 Nov 2017 14:19:58 -0800
Subject: [PATCH 141/297] Add note to keystore docks on requirement for restart
 to take effect (#27488)

---
 docs/reference/setup/secure-settings.asciidoc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/reference/setup/secure-settings.asciidoc b/docs/reference/setup/secure-settings.asciidoc
index 76f7de6b9c1..933e649da58 100644
--- a/docs/reference/setup/secure-settings.asciidoc
+++ b/docs/reference/setup/secure-settings.asciidoc
@@ -11,6 +11,9 @@ NOTE: All commands here should be run as the user which will run Elasticsearch.
 NOTE: Only some settings are designed to be read from the keystore. See
 documentation for each setting to see if it is supported as part of the keystore.
 
+NOTE: All the modifications to the keystore take affect only after restarting
+Elasticsearch.
+
 [float]
 [[creating-keystore]]
 === Creating the keystore

From 756e1706741d1d698319f3b5a14691f15192f1b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E4=BD=9B=E9=99=80=2ERML?= 
Date: Fri, 1 Dec 2017 17:42:42 +0800
Subject: [PATCH 142/297] [Docs] Fix order of nodes usage example  (#27611)

---
 docs/reference/cluster/nodes-usage.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/cluster/nodes-usage.asciidoc b/docs/reference/cluster/nodes-usage.asciidoc
index 5bc57461c8d..d0a5a028966 100644
--- a/docs/reference/cluster/nodes-usage.asciidoc
+++ b/docs/reference/cluster/nodes-usage.asciidoc
@@ -9,8 +9,8 @@ of features for each node.
 
 [source,js]
 --------------------------------------------------
-GET _nodes/nodeId1,nodeId2/usage
 GET _nodes/usage
+GET _nodes/nodeId1,nodeId2/usage
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:node]

From 6cda5b292c1ba5487e314a35b47417a8b5ba2d5f Mon Sep 17 00:00:00 2001
From: Martijn van Groningen 
Date: Wed, 29 Nov 2017 15:46:03 +0100
Subject: [PATCH 143/297] docs: add paragraph about using `percolate` query in
 a filter context

---
 .../query-dsl/percolate-query.asciidoc        | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index a2c91d697fd..410475f42e9 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -144,6 +144,42 @@ In that case the `document` parameter can be substituted with the following para
 `preference`:: Optionally, preference to be used to fetch document to percolate.
 `version`:: Optionally, the expected version of the document to be fetched.
 
+[float]
+==== Percolating in a filter context
+
+In case you are not interested in the score, better performance can be expected by wrapping
+the percolator query in a `bool` query's filter clause or in a `constant_score` query:
+
+[source,js]
+--------------------------------------------------
+GET /my-index/_search
+{
+    "query" : {
+        "constant_score": {
+            "filter": {
+                "percolate" : {
+                    "field" : "query",
+                    "document" : {
+                        "message" : "A new bonsai tree in the office"
+                    }
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+At index time terms are extracted from the percolator query and the percolator
+can often determine whether a query matches just by looking at those extracted
+terms. However, computing scores requires to deserialize each matching query
+and run it against the percolated document, which is a much more expensive
+operation. Hence if computing scores is not required the `percolate` query
+should be wrapped in a `constant_score` query or a `bool` query's filter clause.
+
+Note that the `percolate` query never gets cached by the query cache.
+
 [float]
 ==== Percolating multiple documents
 

From 3e8ca38fcaaa2e92d14baa09ac5aed8e560756de Mon Sep 17 00:00:00 2001
From: Luca Cavanna 
Date: Fri, 1 Dec 2017 12:24:16 +0100
Subject: [PATCH 144/297] Deprecate the transport client in favour of the
 high-level REST client (#27085)

---
 .../transport/client/PreBuiltTransportClient.java         | 4 ++++
 .../elasticsearch/client/transport/TransportClient.java   | 4 ++++
 docs/java-api/client.asciidoc                             | 7 ++-----
 docs/java-api/index.asciidoc                              | 8 +++-----
 .../org/elasticsearch/transport/MockTransportClient.java  | 2 +-
 5 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
index 2c28253e3f1..7f9bcc6ea08 100644
--- a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
+++ b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
@@ -45,8 +45,12 @@ import java.util.concurrent.TimeUnit;
  * {@link MustachePlugin},
  * {@link ParentJoinPlugin}
  * plugins for the client. These plugins are all the required modules for Elasticsearch.
+ *
+ * @deprecated {@link TransportClient} is deprecated in favour of the High Level REST client and will
+ * be removed in Elasticsearch 8.0.
  */
 @SuppressWarnings({"unchecked","varargs"})
+@Deprecated
 public class PreBuiltTransportClient extends TransportClient {
 
     static {
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index 697e0373fb1..182c2532214 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -79,7 +79,11 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
  * 
    
  * The transport client important modules used is the {@link org.elasticsearch.common.network.NetworkModule} which is
  * started in client mode (only connects, no bind).
+ *
+ * @deprecated {@link TransportClient} is deprecated in favour of the High Level REST client and will
+ * be removed in Elasticsearch 8.0.
  */
+@Deprecated
 public abstract class TransportClient extends AbstractClient {
 
     public static final Setting CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL =
diff --git a/docs/java-api/client.asciidoc b/docs/java-api/client.asciidoc
index d5893ffce95..811d7c398d9 100644
--- a/docs/java-api/client.asciidoc
+++ b/docs/java-api/client.asciidoc
@@ -23,14 +23,11 @@ cluster.
 
 ==============================
 
-WARNING: The `TransportClient` is aimed to be replaced by the Java High Level REST
-Client, which executes HTTP requests instead of serialized Java requests. The
-`TransportClient` will be deprecated in upcoming versions of Elasticsearch and it
-is advised to use the Java High Level REST Client instead.
-
 [[transport-client]]
 === Transport Client
 
+deprecated[7.0.0, The `TransportClient` is deprecated in favour of the {java-rest}/java-rest-high.html[Java High Level REST Client] and will be removed in Elasticsearch 8.0. The {java-rest}/java-rest-high-level-migration.html[migration guide] describes all the steps needed to migrate.]
+
 The `TransportClient` connects remotely to an Elasticsearch cluster
 using the transport module. It does not join the cluster, but simply
 gets one or more initial transport addresses and communicates with them
diff --git a/docs/java-api/index.asciidoc b/docs/java-api/index.asciidoc
index 75e97f8a45a..251efff5f8f 100644
--- a/docs/java-api/index.asciidoc
+++ b/docs/java-api/index.asciidoc
@@ -5,6 +5,9 @@ include::../Versions.asciidoc[]
 
 [preface]
 == Preface
+
+deprecated[7.0.0, The `TransportClient` is deprecated in favour of the {java-rest}/java-rest-high.html[Java High Level REST Client] and will be removed in Elasticsearch 8.0. The {java-rest}/java-rest-high-level-migration.html[migration guide] describes all the steps needed to migrate.]
+
 This section describes the Java API that Elasticsearch provides. All
 Elasticsearch operations are executed using a
 <> object. All
@@ -17,11 +20,6 @@ Additionally, operations on a client may be accumulated and executed in
 Note, all the APIs are exposed through the
 Java API (actually, the Java API is used internally to execute them).
 
-WARNING: Starting from version 5.6.0, a new Java client has been
-released: the {java-rest}/java-rest-high.html[Java High Level REST Client].
-This new client is designed to replace the `TransportClient` in Java
-applications which will be deprecated in future versions of Elasticsearch.
-
 == Javadoc
 
 The javadoc for the transport client can be found at {transport-client-javadoc}/index.html.
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java
index 027b26e32d7..ac8c7c6e997 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java
@@ -30,11 +30,11 @@ import java.util.Arrays;
 import java.util.Collection;
 
 @SuppressWarnings({"unchecked","varargs"})
+@Deprecated
 public class MockTransportClient extends TransportClient {
     private static final Settings DEFAULT_SETTINGS = Settings.builder().put("transport.type.default",
         MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME).build();
 
-
     public MockTransportClient(Settings settings, Class... plugins) {
         this(settings, Arrays.asList(plugins));
     }

From f123785d342e33c8888b8b3387bbf6f0cf0351ea Mon Sep 17 00:00:00 2001
From: David Roberts 
Date: Fri, 1 Dec 2017 12:42:09 +0000
Subject: [PATCH 145/297] Add note on how to avoid Jar Hell in IntelliJ 2017.3
 to contributing guide (#27614)

When running unit tests direct from the IDE this setting change is needed
in addition to the idea.no.launcher property that previous versions of IntelliJ
needed.
---
 CONTRIBUTING.md | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index cc0d2ba6e35..a0655b4b849 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -110,17 +110,20 @@ IntelliJ users can automatically configure their IDE: `gradle idea`
 then `File->New Project From Existing Sources`. Point to the root of
 the source directory, select
 `Import project from external model->Gradle`, enable
-`Use auto-import`. Additionally, in order to run tests directly from 
+`Use auto-import`. In order to run tests directly from
 IDEA 2017.2 and above it is required to disable IDEA run launcher to avoid
 finding yourself in "jar hell", which can be achieved by adding the
 `-Didea.no.launcher=true` [JVM
 option](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties)
 or by adding `idea.no.launcher=true` to the
-`idea.properties`[https://www.jetbrains.com/help/idea/file-idea-properties.html]
+[`idea.properties`](https://www.jetbrains.com/help/idea/file-idea-properties.html)
 file which can be accessed under Help > Edit Custom Properties within IDEA. You
 may also need to [remove `ant-javafx.jar` from your
-classpath][https://github.com/elastic/elasticsearch/issues/14348] if that is
-reported as a source of jar hell.
+classpath](https://github.com/elastic/elasticsearch/issues/14348) if that is
+reported as a source of jar hell. Additionally, in order to run tests directly
+from IDEA 2017.3 and above, go to `Run->Edit Configurations...` and change the
+value for the `Shorten command line` setting from `user-local default: none` to
+`classpath file`.
 
 The Elasticsearch codebase makes heavy use of Java `assert`s and the
 test runner requires that assertions be enabled within the JVM. This

From 5060007d20522e570208a8df17bef8a23129d570 Mon Sep 17 00:00:00 2001
From: David Turner 
Date: Fri, 1 Dec 2017 13:33:23 +0000
Subject: [PATCH 146/297] Fix sporadic failures in testCorruptedShards (#27613)

Add assertBusy() to retry in case the shards are not yet all failed, and remove `@AwaitsFix`.

Resolves #12416.
---
 .../shards/IndicesShardStoreRequestIT.java    | 29 ++++++++++---------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
index be8a7894387..2001cb3f8c1 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
@@ -152,7 +152,6 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
         assertThat(shardStatuses.get(index1).size(), equalTo(2));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/12416")
     public void testCorruptedShards() throws Exception {
         String index = "test";
         internalCluster().ensureAtLeastNumDataNodes(2);
@@ -189,22 +188,24 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
             }
         }
 
-        IndicesShardStoresResponse rsp = client().admin().indices().prepareShardStores(index).setShardStatuses("all").get();
-        ImmutableOpenIntMap> shardStatuses = rsp.getStoreStatuses().get(index);
-        assertNotNull(shardStatuses);
-        assertThat(shardStatuses.size(), greaterThan(0));
-        for (IntObjectCursor> shardStatus : shardStatuses) {
-            for (IndicesShardStoresResponse.StoreStatus status : shardStatus.value) {
-                if (corruptedShardIDMap.containsKey(shardStatus.key)
+        assertBusy(() -> { // IndicesClusterStateService#failAndRemoveShard() called asynchronously but we need it to have completed here.
+            IndicesShardStoresResponse rsp = client().admin().indices().prepareShardStores(index).setShardStatuses("all").get();
+            ImmutableOpenIntMap> shardStatuses = rsp.getStoreStatuses().get(index);
+            assertNotNull(shardStatuses);
+            assertThat(shardStatuses.size(), greaterThan(0));
+            for (IntObjectCursor> shardStatus : shardStatuses) {
+                for (IndicesShardStoresResponse.StoreStatus status : shardStatus.value) {
+                    if (corruptedShardIDMap.containsKey(shardStatus.key)
                         && corruptedShardIDMap.get(shardStatus.key).contains(status.getNode().getName())) {
-                    assertThat("shard [" + shardStatus.key + "] is failed on node [" + status.getNode().getName() + "]",
-                        status.getStoreException(), notNullValue());
-                } else {
-                    assertNull("shard [" + shardStatus.key + "] is not failed on node [" + status.getNode().getName() + "]",
-                        status.getStoreException());
+                        assertThat("shard [" + shardStatus.key + "] is failed on node [" + status.getNode().getName() + "]",
+                            status.getStoreException(), notNullValue());
+                    } else {
+                        assertNull("shard [" + shardStatus.key + "] is not failed on node [" + status.getNode().getName() + "]",
+                            status.getStoreException());
+                    }
                 }
             }
-        }
+        });
         logger.info("--> enable allocation");
         enableAllocation(index);
     }

From 623d3700f005e55ebe50ddf73513935562f67721 Mon Sep 17 00:00:00 2001
From: Lee Hinman 
Date: Fri, 1 Dec 2017 07:59:45 -0700
Subject: [PATCH 147/297] Add accounting circuit breaker and track segment
 memory usage (#27116)

* Add accounting circuit breaker and track segment memory usage

This commit adds a new circuit breaker "accounting" that is used for tracking
the memory usage of non-request-tied memory users. It also adds tracking for the
amount of Lucene segment memory used by a shard as a user of the new circuit
breaker.

The Lucene segment memory is updated when the shard refreshes, and removed when
the shard relocates away from a node or is deleted. It should also be noted that
all tracking for segment memory uses `addWithoutBreaking` so as not to fail the
shard if a limit is reached.

The `accounting` breaker has a default limit of 100% and will contribute to the
parent breaker limit.

Resolves #27044
---
 .../common/breaker/CircuitBreaker.java        |  28 ++++
 .../common/settings/ClusterSettings.java      |   2 +
 .../org/elasticsearch/index/IndexService.java |   5 +-
 .../index/engine/EngineConfig.java            |  14 +-
 .../index/engine/InternalEngine.java          |   3 +-
 .../engine/RamAccountingSearcherFactory.java  |  84 ++++++++++
 .../elasticsearch/index/shard/IndexShard.java |   9 +-
 .../HierarchyCircuitBreakerService.java       |  15 ++
 .../index/engine/InternalEngineTests.java     |   7 +-
 .../index/shard/IndexShardIT.java             |  76 ++++++++-
 .../index/shard/IndexShardTests.java          | 149 ++++++++++++++++++
 .../index/shard/RefreshListenersTests.java    |   3 +-
 .../IndexingMemoryControllerTests.java        |   3 +-
 .../modules/indices/circuit_breaker.asciidoc  |  18 +++
 .../index/engine/EngineTestCase.java          |   5 +-
 .../index/shard/IndexShardTestCase.java       |   9 +-
 .../test/ExternalTestCluster.java             |   2 +
 .../test/InternalTestCluster.java             |   2 +
 18 files changed, 418 insertions(+), 16 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/index/engine/RamAccountingSearcherFactory.java

diff --git a/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreaker.java b/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreaker.java
index 4f9c4e458c6..0671091d503 100644
--- a/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreaker.java
+++ b/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreaker.java
@@ -28,10 +28,38 @@ import java.util.Locale;
  */
 public interface CircuitBreaker {
 
+    /**
+     * The parent breaker is a sum of all the following breakers combined. With
+     * this we allow a single breaker to have a significant amount of memory
+     * available while still having a "total" limit for all breakers. Note that
+     * it's not a "real" breaker in that it cannot be added to or subtracted
+     * from by itself.
+     */
     String PARENT = "parent";
+    /**
+     * The fielddata breaker tracks data used for fielddata (on fields) as well
+     * as the id cached used for parent/child queries.
+     */
     String FIELDDATA = "fielddata";
+    /**
+     * The request breaker tracks memory used for particular requests. This
+     * includes allocations for things like the cardinality aggregation, and
+     * accounting for the number of buckets used in an aggregation request.
+     * Generally the amounts added to this breaker are released after a request
+     * is finished.
+     */
     String REQUEST = "request";
+    /**
+     * The in-flight request breaker tracks bytes allocated for reading and
+     * writing requests on the network layer.
+     */
     String IN_FLIGHT_REQUESTS = "in_flight_requests";
+    /**
+     * The accounting breaker tracks things held in memory that is independent
+     * of the request lifecycle. This includes memory used by Lucene for
+     * segments.
+     */
+    String ACCOUNTING = "accounting";
 
     enum Type {
         // A regular or child MemoryCircuitBreaker
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 12a95a688ae..2bea2a59e16 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -258,6 +258,8 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING,
                     HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING,
                     HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING,
+                    HierarchyCircuitBreakerService.ACCOUNTING_CIRCUIT_BREAKER_LIMIT_SETTING,
+                    HierarchyCircuitBreakerService.ACCOUNTING_CIRCUIT_BREAKER_OVERHEAD_SETTING,
                     ClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
                     SearchService.DEFAULT_SEARCH_TIMEOUT_SETTING,
                     ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index 78489965e39..2d5a1dda464 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -129,6 +129,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final BigArrays bigArrays;
     private final ScriptService scriptService;
     private final Client client;
+    private final CircuitBreakerService circuitBreakerService;
     private Supplier indexSortSupplier;
 
     public IndexService(
@@ -158,6 +159,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         this.xContentRegistry = xContentRegistry;
         this.similarityService = similarityService;
         this.namedWriteableRegistry = namedWriteableRegistry;
+        this.circuitBreakerService = circuitBreakerService;
         this.mapperService = new MapperService(indexSettings, registry.build(indexSettings), xContentRegistry, similarityService,
             mapperRegistry,
             // we parse all percolator queries as they would be parsed on shard 0
@@ -380,7 +382,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
             indexShard = new IndexShard(routing, this.indexSettings, path, store, indexSortSupplier,
                 indexCache, mapperService, similarityService, engineFactory,
                 eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer,
-                    searchOperationListeners, indexingOperationListeners, () -> globalCheckpointSyncer.accept(shardId));
+                searchOperationListeners, indexingOperationListeners, () -> globalCheckpointSyncer.accept(shardId),
+                circuitBreakerService);
             eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
             eventListener.afterIndexShardCreated(indexShard);
             shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
index fbc87f2279b..f923abc1a6c 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
@@ -39,6 +39,7 @@ import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
 import org.elasticsearch.indices.IndexingMemoryController;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
@@ -73,6 +74,8 @@ public final class EngineConfig {
     private final Sort indexSort;
     private final boolean forceNewHistoryUUID;
     private final TranslogRecoveryRunner translogRecoveryRunner;
+    @Nullable
+    private final CircuitBreakerService circuitBreakerService;
 
     /**
      * Index setting to change the low level lucene codec used for writing new segments.
@@ -118,7 +121,7 @@ public final class EngineConfig {
                         QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
                         boolean forceNewHistoryUUID, TranslogConfig translogConfig, TimeValue flushMergesAfter,
                         List refreshListeners, Sort indexSort,
-                        TranslogRecoveryRunner translogRecoveryRunner) {
+                        TranslogRecoveryRunner translogRecoveryRunner, CircuitBreakerService circuitBreakerService) {
         if (openMode == null) {
             throw new IllegalArgumentException("openMode must not be null");
         }
@@ -147,6 +150,7 @@ public final class EngineConfig {
         this.refreshListeners = refreshListeners;
         this.indexSort = indexSort;
         this.translogRecoveryRunner = translogRecoveryRunner;
+        this.circuitBreakerService = circuitBreakerService;
     }
 
     /**
@@ -358,4 +362,12 @@ public final class EngineConfig {
     public Sort getIndexSort() {
         return indexSort;
     }
+
+    /**
+     * Returns the circuit breaker service for this engine, or {@code null} if none is to be used.
+     */
+    @Nullable
+    public CircuitBreakerService getCircuitBreakerService() {
+        return this.circuitBreakerService;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 82461ca4a99..e431bfb7a5b 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -531,7 +531,8 @@ public class InternalEngine extends Engine {
         try {
             try {
                 final DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId);
-                internalSearcherManager = new SearcherManager(directoryReader, new SearcherFactory());
+                internalSearcherManager = new SearcherManager(directoryReader,
+                        new RamAccountingSearcherFactory(engineConfig.getCircuitBreakerService()));
                 lastCommittedSegmentInfos = readLastCommittedSegmentInfos(internalSearcherManager, store);
                 ExternalSearcherManager externalSearcherManager = new ExternalSearcherManager(internalSearcherManager,
                     externalSearcherFactory);
diff --git a/core/src/main/java/org/elasticsearch/index/engine/RamAccountingSearcherFactory.java b/core/src/main/java/org/elasticsearch/index/engine/RamAccountingSearcherFactory.java
new file mode 100644
index 00000000000..7972d426fba
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/engine/RamAccountingSearcherFactory.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.engine;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.SearcherFactory;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Searcher factory extending {@link EngineSearcherFactory} that tracks the
+ * amount of memory used by segments in the accounting circuit breaker.
+ */
+final class RamAccountingSearcherFactory extends SearcherFactory {
+
+    private final CircuitBreakerService breakerService;
+
+    RamAccountingSearcherFactory(CircuitBreakerService breakerService) {
+        this.breakerService = breakerService;
+    }
+
+    @Override
+    public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException {
+        final CircuitBreaker breaker = breakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+
+        // Construct a list of the previous segment readers, we only want to track memory used
+        // by new readers, so these will be exempted from the circuit breaking accounting.
+        //
+        // The Core CacheKey is used as the key for the set so that deletions still keep the correct
+        // accounting, as using the Reader or Reader's CacheKey causes incorrect accounting.
+        final Set prevReaders;
+        if (previousReader == null) {
+            prevReaders = Collections.emptySet();
+        } else {
+            final List previousReaderLeaves = previousReader.leaves();
+            prevReaders = new HashSet<>(previousReaderLeaves.size());
+            for (LeafReaderContext lrc : previousReaderLeaves) {
+                prevReaders.add(Lucene.segmentReader(lrc.reader()).getCoreCacheHelper().getKey());
+            }
+        }
+
+        for (LeafReaderContext lrc : reader.leaves()) {
+            final SegmentReader segmentReader = Lucene.segmentReader(lrc.reader());
+            // don't add the segment's memory unless it is not referenced by the previous reader
+            // (only new segments)
+            if (prevReaders.contains(segmentReader.getCoreCacheHelper().getKey()) == false) {
+                final long ramBytesUsed = segmentReader.ramBytesUsed();
+                // add the segment memory to the breaker (non-breaking)
+                breaker.addWithoutBreaking(ramBytesUsed);
+                // and register a listener for when the segment is closed to decrement the
+                // breaker accounting
+                segmentReader.getCoreCacheHelper().addClosedListener(k -> breaker.addWithoutBreaking(-ramBytesUsed));
+            }
+        }
+        return super.newSearcher(reader, previousReader);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 0f554b6ace1..1dc28915d09 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -125,6 +125,7 @@ import org.elasticsearch.index.warmer.WarmerStats;
 import org.elasticsearch.indices.IndexingMemoryController;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.TypeMissingException;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
@@ -187,6 +188,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     private final IndexEventListener indexEventListener;
     private final QueryCachingPolicy cachingPolicy;
     private final Supplier indexSortSupplier;
+    // Package visible for testing
+    final CircuitBreakerService circuitBreakerService;
 
     private final SearchOperationListener searchOperationListener;
 
@@ -258,7 +261,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             Engine.Warmer warmer,
             List searchOperationListener,
             List listeners,
-            Runnable globalCheckpointSyncer) throws IOException {
+            Runnable globalCheckpointSyncer,
+            CircuitBreakerService circuitBreakerService) throws IOException {
         super(shardRouting.shardId(), indexSettings);
         assert shardRouting.initializing();
         this.shardRouting = shardRouting;
@@ -289,6 +293,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         this.shardBitsetFilterCache = new ShardBitsetFilterCache(shardId, indexSettings);
         state = IndexShardState.CREATED;
         this.path = path;
+        this.circuitBreakerService = circuitBreakerService;
         /* create engine config */
         logger.debug("state: [CREATED]");
 
@@ -2181,7 +2186,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             indexCache.query(), cachingPolicy, forceNewHistoryUUID, translogConfig,
             IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()),
             Arrays.asList(refreshListeners, new RefreshMetricUpdater(refreshMetric)), indexSort,
-            this::runTranslogRecovery);
+            this::runTranslogRecovery, circuitBreakerService);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
index b8ec92ba153..9ea8a3df294 100644
--- a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
+++ b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
@@ -63,6 +63,13 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
     public static final Setting REQUEST_CIRCUIT_BREAKER_TYPE_SETTING =
         new Setting<>("indices.breaker.request.type", "memory", CircuitBreaker.Type::parseValue, Property.NodeScope);
 
+    public static final Setting ACCOUNTING_CIRCUIT_BREAKER_LIMIT_SETTING =
+        Setting.memorySizeSetting("indices.breaker.accounting.limit", "100%", Property.NodeScope);
+    public static final Setting ACCOUNTING_CIRCUIT_BREAKER_OVERHEAD_SETTING =
+        Setting.doubleSetting("indices.breaker.accounting.overhead", 1.0d, 0.0d, Property.NodeScope);
+    public static final Setting ACCOUNTING_CIRCUIT_BREAKER_TYPE_SETTING =
+        new Setting<>("indices.breaker.accounting.type", "memory", CircuitBreaker.Type::parseValue, Property.NodeScope);
+
     public static final Setting IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING =
         Setting.memorySizeSetting("network.breaker.inflight_requests.limit", "100%", Property.Dynamic, Property.NodeScope);
     public static final Setting IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING =
@@ -74,6 +81,7 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
     private volatile BreakerSettings fielddataSettings;
     private volatile BreakerSettings inFlightRequestsSettings;
     private volatile BreakerSettings requestSettings;
+    private volatile BreakerSettings accountingSettings;
 
     // Tripped count for when redistribution was attempted but wasn't successful
     private final AtomicLong parentTripCount = new AtomicLong(0);
@@ -98,6 +106,12 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
                 REQUEST_CIRCUIT_BREAKER_TYPE_SETTING.get(settings)
         );
 
+        this.accountingSettings = new BreakerSettings(CircuitBreaker.ACCOUNTING,
+                ACCOUNTING_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).getBytes(),
+                ACCOUNTING_CIRCUIT_BREAKER_OVERHEAD_SETTING.get(settings),
+                ACCOUNTING_CIRCUIT_BREAKER_TYPE_SETTING.get(settings)
+        );
+
         this.parentSettings = new BreakerSettings(CircuitBreaker.PARENT,
                 TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.get(settings).getBytes(), 1.0,
                 CircuitBreaker.Type.PARENT);
@@ -109,6 +123,7 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
         registerBreaker(this.requestSettings);
         registerBreaker(this.fielddataSettings);
         registerBreaker(this.inFlightRequestsSettings);
+        registerBreaker(this.accountingSettings);
 
         clusterSettings.addSettingsUpdateConsumer(TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING, this::setTotalCircuitBreakerLimit, this::validateTotalCircuitBreakerLimit);
         clusterSettings.addSettingsUpdateConsumer(FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING, this::setFieldDataBreakerLimit);
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index c40923e3c7c..026a01a23c3 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -116,6 +116,7 @@ import org.elasticsearch.index.store.DirectoryUtils;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.hamcrest.MatcherAssert;
 import org.hamcrest.Matchers;
@@ -2546,7 +2547,7 @@ public class InternalEngineTests extends EngineTestCase {
                 threadPool, config.getIndexSettings(), null, store, newMergePolicy(), config.getAnalyzer(), config.getSimilarity(),
                 new CodecService(null, logger), config.getEventListener(), IndexSearcher.getDefaultQueryCache(),
                 IndexSearcher.getDefaultQueryCachingPolicy(), false, translogConfig, TimeValue.timeValueMinutes(5),
-                config.getRefreshListeners(), null, config.getTranslogRecoveryRunner());
+                config.getRefreshListeners(), null, config.getTranslogRecoveryRunner(), new NoneCircuitBreakerService());
 
         try {
             InternalEngine internalEngine = new InternalEngine(brokenConfig);
@@ -2600,7 +2601,7 @@ public class InternalEngineTests extends EngineTestCase {
             threadPool, indexSettings, null, store, newMergePolicy(), config.getAnalyzer(), config.getSimilarity(),
             new CodecService(null, logger), config.getEventListener(), IndexSearcher.getDefaultQueryCache(),
             IndexSearcher.getDefaultQueryCachingPolicy(), false, config.getTranslogConfig(), TimeValue.timeValueMinutes(5),
-            config.getRefreshListeners(), null, config.getTranslogRecoveryRunner());
+            config.getRefreshListeners(), null, config.getTranslogRecoveryRunner(), new NoneCircuitBreakerService());
         engine = new InternalEngine(newConfig);
         if (newConfig.getOpenMode() == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG) {
             engine.recoverFromTranslog();
@@ -2630,7 +2631,7 @@ public class InternalEngineTests extends EngineTestCase {
             threadPool, config.getIndexSettings(), null, store, newMergePolicy(), config.getAnalyzer(), config.getSimilarity(),
             new CodecService(null, logger), config.getEventListener(), IndexSearcher.getDefaultQueryCache(),
             IndexSearcher.getDefaultQueryCachingPolicy(), true, config.getTranslogConfig(), TimeValue.timeValueMinutes(5),
-            config.getRefreshListeners(), null, config.getTranslogRecoveryRunner());
+            config.getRefreshListeners(), null, config.getTranslogRecoveryRunner(), new NoneCircuitBreakerService());
         engine = new InternalEngine(newConfig);
         if (newConfig.getOpenMode() == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG) {
             engine.recoverFromTranslog();
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
index 7c38b7c211f..f0e47b69555 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
@@ -23,8 +23,11 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -40,6 +43,7 @@ import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedRunnable;
+import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
@@ -55,16 +59,21 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.breaker.CircuitBreakerStats;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -94,8 +103,10 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.containsString;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 
 public class IndexShardIT extends ESSingleNodeTestCase {
 
@@ -495,7 +506,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
                 }
             }
         };
-        final IndexShard newShard = newIndexShard(indexService, shard, wrapper, listener);
+        final IndexShard newShard = newIndexShard(indexService, shard, wrapper, getInstanceFromNode(CircuitBreakerService.class), listener);
         shardRef.set(newShard);
         recoverShard(newShard);
 
@@ -506,6 +517,65 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         }
     }
 
+    /** Check that the accounting breaker correctly matches the segments API for memory usage */
+    private void checkAccountingBreaker() {
+        CircuitBreakerService breakerService = getInstanceFromNode(CircuitBreakerService.class);
+        CircuitBreaker acctBreaker = breakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+        long usedMem = acctBreaker.getUsed();
+        assertThat(usedMem, greaterThan(0L));
+        NodesStatsResponse response = client().admin().cluster().prepareNodesStats().setIndices(true).setBreaker(true).get();
+        NodeStats stats = response.getNodes().get(0);
+        assertNotNull(stats);
+        SegmentsStats segmentsStats = stats.getIndices().getSegments();
+        CircuitBreakerStats breakerStats = stats.getBreaker().getStats(CircuitBreaker.ACCOUNTING);
+        assertEquals(usedMem, segmentsStats.getMemoryInBytes());
+        assertEquals(usedMem, breakerStats.getEstimated());
+    }
+
+    public void testCircuitBreakerIncrementedByIndexShard() throws Exception {
+        client().admin().cluster().prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().put("network.breaker.inflight_requests.overhead", 0.0)).get();
+
+        // Generate a couple of segments
+        client().prepareIndex("test", "doc", "1").setSource("{\"foo\":\"" + randomAlphaOfLength(100) + "\"}", XContentType.JSON)
+                .setRefreshPolicy(IMMEDIATE).get();
+        // Use routing so 2 documents are guarenteed to be on the same shard
+        String routing = randomAlphaOfLength(5);
+        client().prepareIndex("test", "doc", "2").setSource("{\"foo\":\"" + randomAlphaOfLength(100) + "\"}", XContentType.JSON)
+                .setRefreshPolicy(IMMEDIATE).setRouting(routing).get();
+        client().prepareIndex("test", "doc", "3").setSource("{\"foo\":\"" + randomAlphaOfLength(100) + "\"}", XContentType.JSON)
+                .setRefreshPolicy(IMMEDIATE).setRouting(routing).get();
+
+        checkAccountingBreaker();
+        // Test that force merging causes the breaker to be correctly adjusted
+        logger.info("--> force merging to a single segment");
+        client().admin().indices().prepareForceMerge("test").setMaxNumSegments(1).setFlush(randomBoolean()).get();
+        client().admin().indices().prepareRefresh().get();
+        checkAccountingBreaker();
+
+        client().admin().cluster().prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().put("indices.breaker.total.limit", "1kb")).get();
+
+        // Test that we're now above the parent limit due to the segments
+        Exception e = expectThrows(Exception.class,
+                () -> client().prepareSearch("test").addAggregation(AggregationBuilders.terms("foo_terms").field("foo.keyword")).get());
+        logger.info("--> got: {}", ExceptionsHelper.detailedMessage(e));
+        assertThat(ExceptionsHelper.detailedMessage(e), containsString("[parent] Data too large, data for []"));
+
+        client().admin().cluster().prepareUpdateSettings()
+                .setTransientSettings(Settings.builder()
+                        .putNull("indices.breaker.total.limit")
+                        .putNull("network.breaker.inflight_requests.overhead")).get();
+
+        // Test that deleting the index causes the breaker to correctly be decremented
+        logger.info("--> deleting index");
+        client().admin().indices().prepareDelete("test").get();
+
+        // Accounting breaker should now be 0
+        CircuitBreakerService breakerService = getInstanceFromNode(CircuitBreakerService.class);
+        CircuitBreaker acctBreaker = breakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+        assertThat(acctBreaker.getUsed(), equalTo(0L));
+    }
 
     public static final IndexShard recoverShard(IndexShard newShard) throws IOException {
         DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
@@ -516,12 +586,12 @@ public class IndexShardIT extends ESSingleNodeTestCase {
     }
 
     public static final IndexShard newIndexShard(IndexService indexService, IndexShard shard, IndexSearcherWrapper wrapper,
-                                                 IndexingOperationListener... listeners) throws IOException {
+                                                 CircuitBreakerService cbs, IndexingOperationListener... listeners) throws IOException {
         ShardRouting initializingShardRouting = getInitializingShardRouting(shard.routingEntry());
         IndexShard newShard = new IndexShard(initializingShardRouting, indexService.getIndexSettings(), shard.shardPath(),
             shard.store(), indexService.getIndexSortSupplier(), indexService.cache(), indexService.mapperService(), indexService.similarityService(),
             shard.getEngineFactory(), indexService.getIndexEventListener(), wrapper,
-            indexService.getThreadPool(), indexService.getBigArrays(), null, Collections.emptyList(), Arrays.asList(listeners), () -> {});
+            indexService.getThreadPool(), indexService.getBigArrays(), null, Collections.emptyList(), Arrays.asList(listeners), () -> {}, cbs);
         return newShard;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index b3a0f4b88de..96bcb9382ee 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -32,6 +32,7 @@ import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
@@ -55,6 +56,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -76,6 +78,8 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineException;
+import org.elasticsearch.index.engine.InternalEngine;
+import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@@ -155,6 +159,7 @@ import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -2732,4 +2737,148 @@ public class IndexShardTests extends IndexShardTestCase {
         latch1.await();
         closeShards(primary);
     }
+
+    public void testSegmentMemoryTrackedInBreaker() throws Exception {
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        IndexMetaData metaData = IndexMetaData.builder("test")
+            .putMapping("test", "{ \"properties\": { \"foo\":  { \"type\": \"text\"}}}")
+            .settings(settings)
+            .primaryTerm(0, 1).build();
+        IndexShard primary = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null);
+        recoverShardFromStore(primary);
+        indexDoc(primary, "test", "0", "{\"foo\" : \"foo\"}");
+        primary.refresh("forced refresh");
+
+        SegmentsStats ss = primary.segmentStats(randomBoolean());
+        CircuitBreaker breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+        assertThat(ss.getMemoryInBytes(), equalTo(breaker.getUsed()));
+        final long preRefreshBytes = ss.getMemoryInBytes();
+
+        indexDoc(primary, "test", "1", "{\"foo\" : \"bar\"}");
+        indexDoc(primary, "test", "2", "{\"foo\" : \"baz\"}");
+        indexDoc(primary, "test", "3", "{\"foo\" : \"eggplant\"}");
+
+        ss = primary.segmentStats(randomBoolean());
+        breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+        assertThat(preRefreshBytes, equalTo(breaker.getUsed()));
+
+        primary.refresh("refresh");
+
+        ss = primary.segmentStats(randomBoolean());
+        breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+        assertThat(breaker.getUsed(), equalTo(ss.getMemoryInBytes()));
+        assertThat(breaker.getUsed(), greaterThan(preRefreshBytes));
+
+        indexDoc(primary, "test", "4", "{\"foo\": \"potato\"}");
+        // Forces a refresh with the INTERNAL scope
+        ((InternalEngine) primary.getEngine()).writeIndexingBuffer();
+
+        ss = primary.segmentStats(randomBoolean());
+        breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+        assertThat(breaker.getUsed(), equalTo(ss.getMemoryInBytes()));
+        assertThat(breaker.getUsed(), greaterThan(preRefreshBytes));
+        final long postRefreshBytes = ss.getMemoryInBytes();
+
+        // Deleting a doc causes its memory to be freed from the breaker
+        deleteDoc(primary, "test", "0");
+        primary.refresh("force refresh");
+
+        ss = primary.segmentStats(randomBoolean());
+        breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+        assertThat(breaker.getUsed(), lessThan(postRefreshBytes));
+
+        closeShards(primary);
+
+        breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+        assertThat(breaker.getUsed(), equalTo(0L));
+    }
+
+    public void testSegmentMemoryTrackedWithRandomSearchers() throws Exception {
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        IndexMetaData metaData = IndexMetaData.builder("test")
+            .putMapping("test", "{ \"properties\": { \"foo\":  { \"type\": \"text\"}}}")
+            .settings(settings)
+            .primaryTerm(0, 1).build();
+        IndexShard primary = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, null);
+        recoverShardFromStore(primary);
+
+        int threadCount = randomIntBetween(2, 6);
+        List threads = new ArrayList<>(threadCount);
+        int iterations = randomIntBetween(50, 100);
+        List searchers = Collections.synchronizedList(new ArrayList<>());
+
+        logger.info("--> running with {} threads and {} iterations each", threadCount, iterations);
+        for (int threadId = 0; threadId < threadCount; threadId++) {
+            final String threadName = "thread-" + threadId;
+            Runnable r = () -> {
+                for (int i = 0; i < iterations; i++) {
+                    try {
+                        if (randomBoolean()) {
+                            String id = "id-" + threadName + "-" + i;
+                            logger.debug("--> {} indexing {}", threadName, id);
+                            indexDoc(primary, "test", id, "{\"foo\" : \"" + randomAlphaOfLength(10) + "\"}");
+                        }
+
+                        if (randomBoolean() && i > 10) {
+                            String id = "id-" + threadName + "-" + randomIntBetween(0, i - 1);
+                            logger.debug("--> {}, deleting {}", threadName, id);
+                            deleteDoc(primary, "test", id);
+                        }
+
+                        if (randomBoolean()) {
+                            logger.debug("--> {} refreshing", threadName);
+                            primary.refresh("forced refresh");
+                        }
+
+                        if (randomBoolean()) {
+                            String searcherName = "searcher-" + threadName + "-" + i;
+                            logger.debug("--> {} acquiring new searcher {}", threadName, searcherName);
+                            // Acquire a new searcher, adding it to the list
+                            searchers.add(primary.acquireSearcher(searcherName));
+                        }
+
+                        if (randomBoolean() && searchers.size() > 1) {
+                            // Close one of the searchers at random
+                            Engine.Searcher searcher = searchers.remove(0);
+                            logger.debug("--> {} closing searcher {}", threadName, searcher.source());
+                            IOUtils.close(searcher);
+                        }
+                    } catch (Exception e) {
+                        logger.warn("--> got exception: ", e);
+                        fail("got an exception we didn't expect");
+                    }
+                }
+
+            };
+            threads.add(new Thread(r, threadName));
+        }
+        threads.stream().forEach(t -> t.start());
+
+        for (Thread t : threads) {
+            t.join();
+        }
+
+        // Close remaining searchers
+        IOUtils.close(searchers);
+
+        SegmentsStats ss = primary.segmentStats(randomBoolean());
+        CircuitBreaker breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+        long segmentMem = ss.getMemoryInBytes();
+        long breakerMem = breaker.getUsed();
+        logger.info("--> comparing segmentMem: {} - breaker: {} => {}", segmentMem, breakerMem, segmentMem == breakerMem);
+        assertThat(segmentMem, equalTo(breakerMem));
+
+        // Close shard
+        closeShards(primary);
+
+        // Check that the breaker was successfully reset to 0, meaning that all the accounting was correctly applied
+        breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+        assertThat(breaker.getUsed(), equalTo(0L));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index 53ced098c04..e3158a21853 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -51,6 +51,7 @@ import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.store.DirectoryService;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.TranslogConfig;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -122,7 +123,7 @@ public class RefreshListenersTests extends ESTestCase {
         EngineConfig config = new EngineConfig(EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG, shardId, allocationId, threadPool,
                 indexSettings, null, store, newMergePolicy(), iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger),
                 eventListener, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), false, translogConfig,
-                TimeValue.timeValueMinutes(5), Collections.singletonList(listeners), null, null);
+                TimeValue.timeValueMinutes(5), Collections.singletonList(listeners), null, null, new NoneCircuitBreakerService());
         engine = new InternalEngine(config);
         listeners.setTranslog(engine.getTranslog());
     }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index 2a490c1dcf9..c079ebea840 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardIT;
 import org.elasticsearch.index.shard.IndexShardTestCase;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -440,7 +441,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
                 shard.writeIndexingBuffer();
             }
         };
-        final IndexShard newShard = IndexShardIT.newIndexShard(indexService, shard, wrapper, imc);
+        final IndexShard newShard = IndexShardIT.newIndexShard(indexService, shard, wrapper, new NoneCircuitBreakerService(), imc);
         shardRef.set(newShard);
         try {
             assertEquals(0, imc.availableShards().size());
diff --git a/docs/reference/modules/indices/circuit_breaker.asciidoc b/docs/reference/modules/indices/circuit_breaker.asciidoc
index 7792e10c684..857f54132cc 100644
--- a/docs/reference/modules/indices/circuit_breaker.asciidoc
+++ b/docs/reference/modules/indices/circuit_breaker.asciidoc
@@ -72,6 +72,24 @@ memory on a node. The memory usage is based on the content length of the request
     A constant that all in flight requests estimations are multiplied with to determine a
     final estimation. Defaults to 1
 
+[[accounting-circuit-breaker]]
+[float]
+==== Accounting requests circuit breaker
+
+The in flight requests circuit breaker allows Elasticsearch to limit the memory
+usage of things held in memory that are not released when a request is
+completed. This includes things like the Lucene segment memory.
+
+`network.breaker.accounting.limit`::
+
+    Limit for accounting breaker, defaults to 100% of JVM heap. This means that it is bound
+    by the limit configured for the parent circuit breaker.
+
+`network.breaker.accounting.overhead`::
+
+    A constant that all accounting estimations are multiplied with to determine a
+    final estimation. Defaults to 1
+
 [[script-compilation-circuit-breaker]]
 [float]
 ==== Script compilation circuit breaker
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
index 5c2ef977b16..9ba6f64d74c 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
@@ -68,6 +68,7 @@ import org.elasticsearch.index.store.DirectoryService;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -162,7 +163,7 @@ public abstract class EngineTestCase extends ESTestCase {
                 config.getWarmer(), config.getStore(), config.getMergePolicy(), analyzer, config.getSimilarity(),
                 new CodecService(null, logger), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(),
                 config.getForceNewHistoryUUID(), config.getTranslogConfig(), config.getFlushMergesAfter(), config.getRefreshListeners(),
-                config.getIndexSort(), config.getTranslogRecoveryRunner());
+                config.getIndexSort(), config.getTranslogRecoveryRunner(), config.getCircuitBreakerService());
     }
 
     @Override
@@ -401,7 +402,7 @@ public abstract class EngineTestCase extends ESTestCase {
         EngineConfig config = new EngineConfig(openMode, shardId, allocationId.getId(), threadPool, indexSettings, null, store,
                 mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), listener,
                 IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), false, translogConfig,
-                TimeValue.timeValueMinutes(5), refreshListenerList, indexSort, handler);
+                TimeValue.timeValueMinutes(5), refreshListenerList, indexSort, handler, new NoneCircuitBreakerService());
 
         return config;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index a6cb3ee3b95..4c3e3fb7b48 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -41,6 +41,7 @@ import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.util.BigArrays;
@@ -64,6 +65,9 @@ import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.index.store.DirectoryService;
 import org.elasticsearch.index.store.Store;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoverySourceHandler;
@@ -289,9 +293,12 @@ public abstract class IndexShardTestCase extends ESTestCase {
             };
             final Engine.Warmer warmer = searcher -> {
             };
+            ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+            CircuitBreakerService breakerService = new HierarchyCircuitBreakerService(nodeSettings, clusterSettings);
             indexShard = new IndexShard(routing, indexSettings, shardPath, store, () -> null, indexCache, mapperService, similarityService,
                 engineFactory, indexEventListener, indexSearcherWrapper, threadPool,
-                BigArrays.NON_RECYCLING_INSTANCE, warmer, Collections.emptyList(), Arrays.asList(listeners), globalCheckpointSyncer);
+                BigArrays.NON_RECYCLING_INSTANCE, warmer, Collections.emptyList(), Arrays.asList(listeners), globalCheckpointSyncer,
+                breakerService);
             success = true;
         } finally {
             if (success == false) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
index f8d9c27fd4d..5eb34d96d69 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
@@ -168,6 +168,8 @@ public final class ExternalTestCluster extends TestCluster {
             for (NodeStats stats : nodeStats.getNodes()) {
                 assertThat("Fielddata breaker not reset to 0 on node: " + stats.getNode(),
                         stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
+                assertThat("Accounting breaker not reset to 0 on node: " + stats.getNode(),
+                        stats.getBreaker().getStats(CircuitBreaker.ACCOUNTING).getEstimated(), equalTo(0L));
                 // ExternalTestCluster does not check the request breaker,
                 // because checking it requires a network request, which in
                 // turn increments the breaker, making it non-0
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 9ad0afdf6a4..ea9b17f10e3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -2050,6 +2050,8 @@ public final class InternalTestCluster extends TestCluster {
                 final CircuitBreakerService breakerService = getInstanceFromNode(CircuitBreakerService.class, nodeAndClient.node);
                 CircuitBreaker fdBreaker = breakerService.getBreaker(CircuitBreaker.FIELDDATA);
                 assertThat("Fielddata breaker not reset to 0 on node: " + name, fdBreaker.getUsed(), equalTo(0L));
+                CircuitBreaker acctBreaker = breakerService.getBreaker(CircuitBreaker.ACCOUNTING);
+                assertThat("Accounting breaker not reset to 0 on node: " + name, acctBreaker.getUsed(), equalTo(0L));
                 // Anything that uses transport or HTTP can increase the
                 // request breaker (because they use bigarrays), because of
                 // that the breaker can sometimes be incremented from ping

From e1aa6e2cda099c4f4d65340bc9bc2f18cbf0a894 Mon Sep 17 00:00:00 2001
From: Clinton Gormley 
Date: Fri, 1 Dec 2017 16:54:30 +0100
Subject: [PATCH 148/297] Fix cluster usage docs test

#27611 broke the docs tests because $node_name in the URL doesn't (#27616)seem to be replaced.

Changing this to a * to match all nodes seems to fix the test
---
 docs/reference/cluster/nodes-usage.asciidoc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/reference/cluster/nodes-usage.asciidoc b/docs/reference/cluster/nodes-usage.asciidoc
index d0a5a028966..2d71e749597 100644
--- a/docs/reference/cluster/nodes-usage.asciidoc
+++ b/docs/reference/cluster/nodes-usage.asciidoc
@@ -14,7 +14,7 @@ GET _nodes/nodeId1,nodeId2/usage
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:node]
-// TEST[s/nodeId1,nodeId2/\$node_name/]
+// TEST[s/nodeId1,nodeId2/*/]
 
 The first command retrieves usage of all the nodes in the cluster. The
 second command selectively retrieves nodes usage of only `nodeId1` and
@@ -25,8 +25,8 @@ second command selectively retrieves nodes usage of only `nodeId1` and
 [[rest-usage]]
 ==== REST actions usage information
 
-The `rest_actions` field in the response contains a map of the REST 
-actions classname with a count of the number of times that action has 
+The `rest_actions` field in the response contains a map of the REST
+actions classname with a count of the number of times that action has
 been called on the node:
 
 [source,js]
@@ -59,6 +59,6 @@ been called on the node:
 // TESTRESPONSE[s/1492553906606/$body.$_path/]
 // TESTRESPONSE[s/"rest_actions": [^}]+}/"rest_actions": $body.$_path/]
 <1> Timestamp for when this nodes usage request was performed.
-<2> Timestamp for when the usage information recording was started. This is 
+<2> Timestamp for when the usage information recording was started. This is
 equivalent to the time that the node was started.
 <3> Search action has been called 19 times for this node.
\ No newline at end of file

From e16f1271b69be0f3286e24ae0a37353dae892dd1 Mon Sep 17 00:00:00 2001
From: James Baiera 
Date: Fri, 1 Dec 2017 14:26:05 -0500
Subject: [PATCH 149/297] Fix SecurityException when HDFS Repository used
 against HA Namenodes (#27196)

* Sense HA HDFS settings and remove permission restrictions during regular execution.

This PR adds integration tests for HA-Enabled HDFS deployments, both regular and secured.
The Mini HDFS fixture has been updated to optionally run in HA-Mode. A new test suite has
been added for reproducing the effects of a Namenode failing over during regular repository
usage. Going forward, the HDFS Repository will still be subject to its self imposed permission
restrictions during normal use, but will no longer restrict them when running against an HA
enabled HDFS cluster. Instead, the plugin will rely on the provided security policy and not
further restrict the permissions so that the transparent operation to failover to a different
Namenode in the client does not raise security exceptions. Additionally, we are now testing the
secure mode with SASL based wire encryption of data between Elasticsearch and HDFS. This
includes a missing library (commons codec) in order to support this change.
---
 plugins/repository-hdfs/build.gradle          | 195 ++++++++----
 .../licenses/commons-codec-1.10.jar.sha1      |   1 +
 .../licenses/commons-codec-LICENSE.txt        | 202 +++++++++++++
 .../licenses/commons-codec-NOTICE.txt         |  17 ++
 .../repositories/hdfs/HdfsBlobContainer.java  |  21 +-
 .../repositories/hdfs/HdfsBlobStore.java      |  28 +-
 .../repositories/hdfs/HdfsRepository.java     |  55 ++--
 .../hdfs/HdfsSecurityContext.java             |  29 +-
 .../hdfs/HaHdfsFailoverTestSuiteIT.java       | 283 ++++++++++++++++++
 .../src/main/java/hdfs/MiniHDFS.java          |  34 ++-
 10 files changed, 740 insertions(+), 125 deletions(-)
 create mode 100644 plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/commons-codec-LICENSE.txt
 create mode 100644 plugins/repository-hdfs/licenses/commons-codec-NOTICE.txt
 create mode 100644 plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java

diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index 6426f588352..19ca4c01482 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -18,6 +18,7 @@
  */
 
 import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.test.ClusterConfiguration
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
 import java.nio.file.Files
@@ -51,6 +52,7 @@ dependencies {
   compile 'com.google.protobuf:protobuf-java:2.5.0'
   compile 'commons-logging:commons-logging:1.1.3'
   compile 'commons-cli:commons-cli:1.2'
+  compile 'commons-codec:commons-codec:1.10'
   compile 'commons-collections:commons-collections:3.2.2'
   compile 'commons-configuration:commons-configuration:1.6'
   compile 'commons-io:commons-io:2.4'
@@ -66,14 +68,6 @@ dependencyLicenses {
   mapping from: /hadoop-.*/, to: 'hadoop'
 }
 
-task hdfsFixture(type: org.elasticsearch.gradle.test.AntFixture) {
-  dependsOn project.configurations.hdfsFixture
-  executable = new File(project.javaHome, 'bin/java')
-  env 'CLASSPATH', "${ -> project.configurations.hdfsFixture.asPath }"
-  args 'hdfs.MiniHDFS',
-       baseDir
-}
-
 // MIT Kerberos Vagrant Testing Fixture
 String box = "krb5kdc"
 Map vagrantEnvVars = [
@@ -116,28 +110,106 @@ for (String principal : principals) {
   krb5AddPrincipals.dependsOn(create)
 }
 
-task secureHdfsFixture(type: org.elasticsearch.gradle.test.AntFixture) {
-  dependsOn project.configurations.hdfsFixture, krb5kdcFixture, krb5AddPrincipals
-  executable = new File(project.javaHome, 'bin/java')
-  env 'CLASSPATH', "${ -> project.configurations.hdfsFixture.asPath }"
+// Create HDFS File System Testing Fixtures for HA/Secure combinations
+for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', 'secureHaHdfsFixture']) {
+  project.tasks.create(fixtureName, org.elasticsearch.gradle.test.AntFixture) {
+    dependsOn project.configurations.hdfsFixture
+    executable = new File(project.javaHome, 'bin/java')
+    env 'CLASSPATH', "${ -> project.configurations.hdfsFixture.asPath }"
 
-  Path keytabPath = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("hdfs_hdfs.build.elastic.co.keytab")
-  Path krb5Config = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf")
+    final List miniHDFSArgs = []
 
-  final List miniHDFSArgs = ["-Djava.security.krb5.conf=${krb5Config}"]
+    // If it's a secure fixture, then depend on Kerberos Fixture and principals + add the krb5conf to the JVM options
+    if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) {
+      dependsOn krb5kdcFixture, krb5AddPrincipals
+      Path krb5Config = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf")
+      miniHDFSArgs.add("-Djava.security.krb5.conf=${krb5Config}");
+      if (project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_9) {
+        miniHDFSArgs.add('--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED')
+      }
+    }
 
-  if (project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_9) {
-    miniHDFSArgs.add('--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED')
+    // If it's an HA fixture, set a nameservice to use in the JVM options
+    if (fixtureName.equals('haHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) {
+      miniHDFSArgs.add("-Dha-nameservice=ha-hdfs")
+    }
+
+    // Common options
+    miniHDFSArgs.add('hdfs.MiniHDFS')
+    miniHDFSArgs.add(baseDir)
+
+    // If it's a secure fixture, then set the principal name and keytab locations to use for auth.
+    if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) {
+      Path keytabPath = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("hdfs_hdfs.build.elastic.co.keytab")
+      miniHDFSArgs.add("hdfs/hdfs.build.elastic.co@${realm}")
+      miniHDFSArgs.add("${keytabPath}")
+    }
+
+    args miniHDFSArgs.toArray()
   }
-
-  miniHDFSArgs.add('hdfs.MiniHDFS')
-  miniHDFSArgs.add(baseDir)
-  miniHDFSArgs.add("hdfs/hdfs.build.elastic.co@${realm}")
-  miniHDFSArgs.add("${keytabPath}")
-
-  args miniHDFSArgs.toArray()
 }
 
+// The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
+project.afterEvaluate {
+  for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) {
+    ClusterConfiguration cluster = project.extensions.getByName("${integTestTaskName}Cluster") as ClusterConfiguration
+    cluster.dependsOn(project.bundlePlugin)
+
+    Task restIntegTestTask = project.tasks.getByName(integTestTaskName)
+    restIntegTestTask.clusterConfig.plugin(project.path)
+
+    // Default jvm arguments for all test clusters
+    String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
+            " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
+            " " + System.getProperty('tests.jvm.argline', '')
+
+    // If it's a secure cluster, add the keytab as an extra config, and set the krb5 conf in the JVM options.
+    if (integTestTaskName.equals('integTestSecure') || integTestTaskName.equals('integTestSecureHa')) {
+      Path elasticsearchKT = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("elasticsearch.keytab").toAbsolutePath()
+      Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
+
+      restIntegTestTask.clusterConfig.extraConfigFile("repository-hdfs/krb5.keytab", "${elasticsearchKT}")
+      jvmArgs = jvmArgs + " " + "-Djava.security.krb5.conf=${krb5conf}"
+      if (project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_9) {
+        jvmArgs = jvmArgs + " " + '--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED'
+      }
+
+      // If it's the HA + Secure tests then also set the Kerberos settings for the integration test JVM since we'll
+      // need to auth to HDFS to trigger namenode failovers.
+      if (integTestTaskName.equals('integTestSecureHa')) {
+        Task restIntegTestTaskRunner = project.tasks.getByName("${integTestTaskName}Runner")
+        restIntegTestTaskRunner.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}"
+        restIntegTestTaskRunner.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}"
+        restIntegTestTaskRunner.jvmArg "-Djava.security.krb5.conf=${krb5conf}"
+        if (project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_9) {
+          restIntegTestTaskRunner.jvmArg '--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED'
+        }
+
+        Path hdfsKT = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("hdfs_hdfs.build.elastic.co.keytab").toAbsolutePath()
+        restIntegTestTaskRunner.systemProperty "test.krb5.keytab.hdfs", "${hdfsKT}"
+      }
+    }
+
+    restIntegTestTask.clusterConfig.jvmArgs = jvmArgs
+  }
+}
+
+// Create a Integration Test suite just for HA based tests
+RestIntegTestTask integTestHa = project.tasks.create('integTestHa', RestIntegTestTask.class) {
+  description = "Runs rest tests against an elasticsearch cluster with HDFS configured with HA Namenode."
+}
+
+// Create a Integration Test suite just for security based tests
+RestIntegTestTask integTestSecure = project.tasks.create('integTestSecure', RestIntegTestTask.class) {
+  description = "Runs rest tests against an elasticsearch cluster with HDFS secured by MIT Kerberos."
+}
+
+// Create a Integration Test suite just for HA related security based tests
+RestIntegTestTask integTestSecureHa = project.tasks.create('integTestSecureHa', RestIntegTestTask.class) {
+  description = "Runs rest tests against an elasticsearch cluster with HDFS configured with HA Namenode and secured by MIT Kerberos."
+}
+
+// Determine HDFS Fixture compatibility for the current build environment.
 boolean fixtureSupported = false
 if (Os.isFamily(Os.FAMILY_WINDOWS)) {
   // hdfs fixture will not start without hadoop native libraries on windows
@@ -145,9 +217,9 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
   if (nativePath != null) {
     Path path = Paths.get(nativePath);
     if (Files.isDirectory(path) &&
-        Files.exists(path.resolve("bin").resolve("winutils.exe")) &&
-        Files.exists(path.resolve("bin").resolve("hadoop.dll")) &&
-        Files.exists(path.resolve("bin").resolve("hdfs.dll"))) {
+            Files.exists(path.resolve("bin").resolve("winutils.exe")) &&
+            Files.exists(path.resolve("bin").resolve("hadoop.dll")) &&
+            Files.exists(path.resolve("bin").resolve("hdfs.dll"))) {
       fixtureSupported = true
     } else {
       throw new IllegalStateException("HADOOP_HOME: ${path} is invalid, does not contain hadoop native libraries in \$HADOOP_HOME/bin");
@@ -157,56 +229,63 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
   fixtureSupported = true
 }
 
+// Always ignore HA integration tests in the normal integration test runner, they are included below as
+// part of their own HA-specific integration test tasks.
+integTestRunner.exclude('**/Ha*TestSuiteIT.class')
+
 if (fixtureSupported) {
+  // Check depends on the HA test. Already depends on the standard test.
+  project.check.dependsOn(integTestHa)
+
+  // Both standard and HA tests depend on their respective HDFS fixtures
   integTestCluster.dependsOn hdfsFixture
+  integTestHaCluster.dependsOn haHdfsFixture
+
+  // The normal test runner only runs the standard hdfs rest tests
   integTestRunner.systemProperty 'tests.rest.suite', 'hdfs_repository'
+
+  // Only include the HA integration tests for the HA test task
+  integTestHaRunner.patternSet.setIncludes(['**/Ha*TestSuiteIT.class'])
 } else {
   logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH")
-  // just tests that the plugin loads
+  // The normal integration test runner will just test that the plugin loads
   integTestRunner.systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic'
+  // HA fixture is unsupported. Don't run them.
+  integTestHa.setEnabled(false)
 }
 
+// Secure HDFS testing relies on the Vagrant based Kerberos fixture.
 boolean secureFixtureSupported = false
 if (fixtureSupported) {
   secureFixtureSupported = project.rootProject.vagrantSupported
 }
 
-// Create a Integration Test suite just for security based tests
-// This must execute before the afterEvaluate block from integTestSecure
-project.afterEvaluate {
-  Path elasticsearchKT = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("elasticsearch.keytab").toAbsolutePath()
-  Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
-
-  project.integTestSecureCluster.dependsOn(project.bundlePlugin)
-  project.integTestSecure.clusterConfig.plugin(project.path)
-  project.integTestSecure.clusterConfig.extraConfigFile("repository-hdfs/krb5.keytab", "${elasticsearchKT}")
-
-  final String baseJvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
-          " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
-          " " + "-Djava.security.krb5.conf=${krb5conf}" +
-          " " + System.getProperty('tests.jvm.argline', '')
-  final String jvmArgs
-  if (project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_9) {
-    jvmArgs = baseJvmArgs + " " + '--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED'
-  } else {
-    jvmArgs = baseJvmArgs
-  }
-
-  project.integTestSecure.clusterConfig.jvmArgs = jvmArgs
-}
-
-RestIntegTestTask integTestSecure = project.tasks.create('integTestSecure', RestIntegTestTask.class) {
-  description = "Runs rest tests against an elasticsearch cluster with HDFS secured by MIT Kerberos."
-}
-
 if (secureFixtureSupported) {
   project.check.dependsOn(integTestSecure)
+  project.check.dependsOn(integTestSecureHa)
 
   // Fixture dependencies
   integTestSecureCluster.dependsOn secureHdfsFixture, krb5kdcFixture
+  integTestSecureHaCluster.dependsOn secureHaHdfsFixture, krb5kdcFixture
+
+  // Set the keytab files in the classpath so that we can access them from test code without the security manager
+  // freaking out.
+  Path hdfsKeytabPath = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs")
+  project.dependencies {
+    testRuntime fileTree(dir: hdfsKeytabPath.toString(), include: ['*.keytab'])
+  }
+
+  // Run just the secure hdfs rest test suite.
   integTestSecureRunner.systemProperty 'tests.rest.suite', 'secure_hdfs_repository'
+  // Ignore HA integration Tests. They are included below as part of integTestSecureHa test runner.
+  integTestSecureRunner.exclude('**/Ha*TestSuiteIT.class')
+
+  // Only include the HA integration tests for the HA test task
+  integTestSecureHaRunner.patternSet.setIncludes(['**/Ha*TestSuiteIT.class'])
 } else {
+  // Security tests unsupported. Don't run these tests.
   integTestSecure.enabled = false
+  integTestSecureHa.enabled = false
 }
 
 thirdPartyAudit.excludes = [
@@ -311,10 +390,6 @@ thirdPartyAudit.excludes = [
   'org.apache.commons.beanutils.PropertyUtils', 
   'org.apache.commons.compress.archivers.tar.TarArchiveEntry', 
   'org.apache.commons.compress.archivers.tar.TarArchiveInputStream', 
-  'org.apache.commons.codec.DecoderException', 
-  'org.apache.commons.codec.binary.Base64', 
-  'org.apache.commons.codec.binary.Hex', 
-  'org.apache.commons.codec.digest.DigestUtils', 
   'org.apache.commons.daemon.Daemon', 
   'org.apache.commons.daemon.DaemonContext', 
   'org.apache.commons.digester.AbstractObjectCreationFactory', 
diff --git a/plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1 b/plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1
new file mode 100644
index 00000000000..3fe8682a1b0
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1
@@ -0,0 +1 @@
+4b95f4897fa13f2cd904aee711aeafc0c5295cd8
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/commons-codec-LICENSE.txt b/plugins/repository-hdfs/licenses/commons-codec-LICENSE.txt
new file mode 100644
index 00000000000..75b52484ea4
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/commons-codec-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-hdfs/licenses/commons-codec-NOTICE.txt b/plugins/repository-hdfs/licenses/commons-codec-NOTICE.txt
new file mode 100644
index 00000000000..efc098ca3ee
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/commons-codec-NOTICE.txt
@@ -0,0 +1,17 @@
+Apache Commons Codec
+Copyright 2002-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+contains test data from http://aspell.net/test/orig/batch0.tab.
+Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
+
+===============================================================================
+
+The content of package org.apache.commons.codec.language.bm has been translated
+from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+with permission from the original authors.
+Original source copyright:
+Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
index f160f4c4ead..fa9cda06589 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
@@ -157,39 +157,30 @@ final class HdfsBlobContainer extends AbstractBlobContainer {
         }
 
         public int read() throws IOException {
-            return doPrivilegedOrThrow(in::read);
+            return securityContext.doPrivilegedOrThrow(in::read);
         }
 
         public int read(byte b[]) throws IOException {
-            return doPrivilegedOrThrow(() -> in.read(b));
+            return securityContext.doPrivilegedOrThrow(() -> in.read(b));
         }
 
         public int read(byte b[], int off, int len) throws IOException {
-            return doPrivilegedOrThrow(() -> in.read(b, off, len));
+            return securityContext.doPrivilegedOrThrow(() -> in.read(b, off, len));
         }
 
         public long skip(long n) throws IOException {
-            return doPrivilegedOrThrow(() -> in.skip(n));
+            return securityContext.doPrivilegedOrThrow(() -> in.skip(n));
         }
 
         public int available() throws IOException {
-            return doPrivilegedOrThrow(() -> in.available());
+            return securityContext.doPrivilegedOrThrow(() -> in.available());
         }
 
         public synchronized void reset() throws IOException {
-            doPrivilegedOrThrow(() -> {
+            securityContext.doPrivilegedOrThrow(() -> {
                 in.reset();
                 return null;
             });
         }
-
-        private  T doPrivilegedOrThrow(PrivilegedExceptionAction action) throws IOException {
-            SpecialPermission.check();
-            try {
-                return AccessController.doPrivileged(action, null, securityContext.getRestrictedExecutionPermissions());
-            } catch (PrivilegedActionException e) {
-                throw (IOException) e.getCause();
-            }
-        }
     }
 }
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java
index fc6922d81f4..fde7657fe31 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java
@@ -23,15 +23,11 @@ import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 
 import java.io.IOException;
-import java.security.AccessController;
-import java.security.PrivilegedActionException;
-import java.security.PrivilegedExceptionAction;
 
 final class HdfsBlobStore implements BlobStore {
 
@@ -43,8 +39,14 @@ final class HdfsBlobStore implements BlobStore {
     private volatile boolean closed;
 
     HdfsBlobStore(FileContext fileContext, String path, int bufferSize, boolean readOnly) throws IOException {
+        this(fileContext, path, bufferSize, readOnly, false);
+    }
+
+    HdfsBlobStore(FileContext fileContext, String path, int bufferSize, boolean readOnly, boolean haEnabled) throws IOException {
         this.fileContext = fileContext;
-        this.securityContext = new HdfsSecurityContext(fileContext.getUgi());
+        // Only restrict permissions if not running with HA
+        boolean restrictPermissions = (haEnabled == false);
+        this.securityContext = new HdfsSecurityContext(fileContext.getUgi(), restrictPermissions);
         this.bufferSize = bufferSize;
         this.root = execute(fileContext1 -> fileContext1.makeQualified(new Path(path)));
         this.readOnly = readOnly;
@@ -112,21 +114,13 @@ final class HdfsBlobStore implements BlobStore {
      * Executes the provided operation against this store
      */
      V execute(Operation operation) throws IOException {
-        SpecialPermission.check();
         if (closed) {
             throw new AlreadyClosedException("HdfsBlobStore is closed: " + this);
         }
-        try {
-            return AccessController.doPrivileged((PrivilegedExceptionAction)
-                () -> {
-                    securityContext.ensureLogin();
-                    return operation.run(fileContext);
-                },
-                null,
-                securityContext.getRestrictedExecutionPermissions());
-        } catch (PrivilegedActionException pae) {
-            throw (IOException) pae.getException();
-        }
+        return securityContext.doPrivilegedOrThrow(() -> {
+            securityContext.ensureLogin();
+            return operation.run(fileContext);
+        });
     }
 
     @Override
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
index 1bf2e47e965..5ef1c7d18d6 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
@@ -26,13 +26,13 @@ import java.net.UnknownHostException;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.AbstractFileSystem;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.io.retry.FailoverProxyProvider;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -85,12 +85,12 @@ public final class HdfsRepository extends BlobStoreRepository {
         }
         URI uri = URI.create(uriSetting);
         if ("hdfs".equalsIgnoreCase(uri.getScheme()) == false) {
-            throw new IllegalArgumentException(
-                    String.format(Locale.ROOT, "Invalid scheme [%s] specified in uri [%s]; only 'hdfs' uri allowed for hdfs snapshot/restore", uri.getScheme(), uriSetting));
+            throw new IllegalArgumentException(String.format(Locale.ROOT,
+                "Invalid scheme [%s] specified in uri [%s]; only 'hdfs' uri allowed for hdfs snapshot/restore", uri.getScheme(), uriSetting));
         }
         if (Strings.hasLength(uri.getPath()) && uri.getPath().equals("/") == false) {
             throw new IllegalArgumentException(String.format(Locale.ROOT,
-                    "Use 'path' option to specify a path [%s], not the uri [%s] for hdfs snapshot/restore", uri.getPath(), uriSetting));
+                "Use 'path' option to specify a path [%s], not the uri [%s] for hdfs snapshot/restore", uri.getPath(), uriSetting));
         }
 
         String pathSetting = getMetadata().settings().get("path");
@@ -99,41 +99,42 @@ public final class HdfsRepository extends BlobStoreRepository {
             throw new IllegalArgumentException("No 'path' defined for hdfs snapshot/restore");
         }
 
-        int bufferSize = getMetadata().settings().getAsBytesSize("buffer_size", DEFAULT_BUFFER_SIZE).bytesAsInt();
-
-        try {
-            // initialize our filecontext
-            SpecialPermission.check();
-            FileContext fileContext = AccessController.doPrivileged((PrivilegedAction)
-                () -> createContext(uri, getMetadata().settings()));
-            blobStore = new HdfsBlobStore(fileContext, pathSetting, bufferSize, isReadOnly());
-            logger.debug("Using file-system [{}] for URI [{}], path [{}]", fileContext.getDefaultFileSystem(), fileContext.getDefaultFileSystem().getUri(), pathSetting);
-        } catch (IOException e) {
-            throw new UncheckedIOException(String.format(Locale.ROOT, "Cannot create HDFS repository for uri [%s]", uri), e);
-        }
+        // initialize our blobstore using elevated privileges.
+        SpecialPermission.check();
+        blobStore = AccessController.doPrivileged((PrivilegedAction) () -> createBlobstore(uri, pathSetting, getMetadata().settings()));
         super.doStart();
     }
 
-    // create hadoop filecontext
-    private FileContext createContext(URI uri, Settings repositorySettings)  {
+    private HdfsBlobStore createBlobstore(URI uri, String path, Settings repositorySettings)  {
         Configuration hadoopConfiguration = new Configuration(repositorySettings.getAsBoolean("load_defaults", true));
         hadoopConfiguration.setClassLoader(HdfsRepository.class.getClassLoader());
         hadoopConfiguration.reloadConfiguration();
 
         final Settings confSettings = repositorySettings.getByPrefix("conf.");
         for (String key : confSettings.keySet()) {
+            LOGGER.debug("Adding configuration to HDFS Client Configuration : {} = {}", key, confSettings.get(key));
             hadoopConfiguration.set(key, confSettings.get(key));
         }
 
-        // Create a hadoop user
-        UserGroupInformation ugi = login(hadoopConfiguration, repositorySettings);
-
         // Disable FS cache
         hadoopConfiguration.setBoolean("fs.hdfs.impl.disable.cache", true);
 
+        // Create a hadoop user
+        UserGroupInformation ugi = login(hadoopConfiguration, repositorySettings);
+
+        // Sense if HA is enabled
+        // HA requires elevated permissions during regular usage in the event that a failover operation
+        // occurs and a new connection is required.
+        String host = uri.getHost();
+        String configKey = HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + host;
+        Class ret = hadoopConfiguration.getClass(configKey, null, FailoverProxyProvider.class);
+        boolean haEnabled = ret != null;
+
+        int bufferSize = repositorySettings.getAsBytesSize("buffer_size", DEFAULT_BUFFER_SIZE).bytesAsInt();
+
         // Create the filecontext with our user information
         // This will correctly configure the filecontext to have our UGI as its internal user.
-        return ugi.doAs((PrivilegedAction) () -> {
+        FileContext fileContext = ugi.doAs((PrivilegedAction) () -> {
             try {
                 AbstractFileSystem fs = AbstractFileSystem.get(uri, hadoopConfiguration);
                 return FileContext.getFileContext(fs, hadoopConfiguration);
@@ -141,6 +142,14 @@ public final class HdfsRepository extends BlobStoreRepository {
                 throw new UncheckedIOException(e);
             }
         });
+
+        logger.debug("Using file-system [{}] for URI [{}], path [{}]", fileContext.getDefaultFileSystem(), fileContext.getDefaultFileSystem().getUri(), path);
+
+        try {
+            return new HdfsBlobStore(fileContext, path, bufferSize, isReadOnly(), haEnabled);
+        } catch (IOException e) {
+            throw new UncheckedIOException(String.format(Locale.ROOT, "Cannot create HDFS repository for uri [%s]", uri), e);
+        }
     }
 
     private UserGroupInformation login(Configuration hadoopConfiguration, Settings repositorySettings) {
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java
index bd16d87d879..4562174230c 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java
@@ -24,17 +24,17 @@ import java.lang.reflect.ReflectPermission;
 import java.net.SocketPermission;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.AccessController;
 import java.security.Permission;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
-import java.util.Locale;
-import java.util.function.Supplier;
 import javax.security.auth.AuthPermission;
 import javax.security.auth.PrivateCredentialPermission;
 import javax.security.auth.kerberos.ServicePermission;
 
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.env.Environment;
 
 /**
@@ -45,8 +45,6 @@ import org.elasticsearch.env.Environment;
  */
 class HdfsSecurityContext {
 
-    private static final Logger LOGGER = Loggers.getLogger(HdfsSecurityContext.class);
-
     private static final Permission[] SIMPLE_AUTH_PERMISSIONS;
     private static final Permission[] KERBEROS_AUTH_PERMISSIONS;
     static {
@@ -104,10 +102,12 @@ class HdfsSecurityContext {
     }
 
     private final UserGroupInformation ugi;
+    private final boolean restrictPermissions;
     private final Permission[] restrictedExecutionPermissions;
 
-    HdfsSecurityContext(UserGroupInformation ugi) {
+    HdfsSecurityContext(UserGroupInformation ugi, boolean restrictPermissions) {
         this.ugi = ugi;
+        this.restrictPermissions = restrictPermissions;
         this.restrictedExecutionPermissions = renderPermissions(ugi);
     }
 
@@ -131,10 +131,23 @@ class HdfsSecurityContext {
         return permissions;
     }
 
-    Permission[] getRestrictedExecutionPermissions() {
+    private Permission[] getRestrictedExecutionPermissions() {
         return restrictedExecutionPermissions;
     }
 
+     T doPrivilegedOrThrow(PrivilegedExceptionAction action) throws IOException {
+        SpecialPermission.check();
+        try {
+            if (restrictPermissions) {
+                return AccessController.doPrivileged(action, null, this.getRestrictedExecutionPermissions());
+            } else {
+                return AccessController.doPrivileged(action);
+            }
+        } catch (PrivilegedActionException e) {
+            throw (IOException) e.getCause();
+        }
+    }
+
     void ensureLogin() {
         if (ugi.isFromKeytab()) {
             try {
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java
new file mode 100644
index 00000000000..ce4fe9b6d3f
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.hdfs;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.BadFencingConfigurationException;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.ha.HAServiceTarget;
+import org.apache.hadoop.ha.NodeFencer;
+import org.apache.hadoop.ha.ZKFCProtocol;
+import org.apache.hadoop.ha.protocolPB.HAServiceProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdfs.tools.DFSHAAdmin;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.http.Header;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.nio.entity.NStringEntity;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.Assert;
+
+/**
+ * Integration test that runs against an HA-Enabled HDFS instance
+ */
+public class HaHdfsFailoverTestSuiteIT extends ESRestTestCase {
+
+    public void testHAFailoverWithRepository() throws Exception {
+        RestClient client = client();
+        Map emptyParams = Collections.emptyMap();
+        Header contentHeader = new BasicHeader("Content-Type", "application/json");
+
+        String esKerberosPrincipal = System.getProperty("test.krb5.principal.es");
+        String hdfsKerberosPrincipal = System.getProperty("test.krb5.principal.hdfs");
+        String kerberosKeytabLocation = System.getProperty("test.krb5.keytab.hdfs");
+        boolean securityEnabled = hdfsKerberosPrincipal != null;
+
+        Configuration hdfsConfiguration = new Configuration();
+        hdfsConfiguration.set("dfs.nameservices", "ha-hdfs");
+        hdfsConfiguration.set("dfs.ha.namenodes.ha-hdfs", "nn1,nn2");
+        hdfsConfiguration.set("dfs.namenode.rpc-address.ha-hdfs.nn1", "localhost:10001");
+        hdfsConfiguration.set("dfs.namenode.rpc-address.ha-hdfs.nn2", "localhost:10002");
+        hdfsConfiguration.set(
+            "dfs.client.failover.proxy.provider.ha-hdfs",
+            "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+        );
+
+        AccessController.doPrivileged((PrivilegedExceptionAction) () -> {
+            if (securityEnabled) {
+                // ensure that keytab exists
+                Path kt = PathUtils.get(kerberosKeytabLocation);
+                if (Files.exists(kt) == false) {
+                    throw new IllegalStateException("Could not locate keytab at " + kerberosKeytabLocation);
+                }
+                if (Files.isReadable(kt) != true) {
+                    throw new IllegalStateException("Could not read keytab at " + kerberosKeytabLocation);
+                }
+                logger.info("Keytab Length: " + Files.readAllBytes(kt).length);
+
+                // set principal names
+                hdfsConfiguration.set("dfs.namenode.kerberos.principal", hdfsKerberosPrincipal);
+                hdfsConfiguration.set("dfs.datanode.kerberos.principal", hdfsKerberosPrincipal);
+                hdfsConfiguration.set("dfs.data.transfer.protection", "authentication");
+
+                SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, hdfsConfiguration);
+                UserGroupInformation.setConfiguration(hdfsConfiguration);
+                UserGroupInformation.loginUserFromKeytab(hdfsKerberosPrincipal, kerberosKeytabLocation);
+            } else {
+                SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.SIMPLE, hdfsConfiguration);
+                UserGroupInformation.setConfiguration(hdfsConfiguration);
+                UserGroupInformation.getCurrentUser();
+            }
+            return null;
+        });
+
+        // Create repository
+        {
+            Response response = client.performRequest("PUT", "/_snapshot/hdfs_ha_repo_read", emptyParams, new NStringEntity(
+                "{" +
+                    "\"type\":\"hdfs\"," +
+                    "\"settings\":{" +
+                        "\"uri\": \"hdfs://ha-hdfs/\",\n" +
+                        "\"path\": \"/user/elasticsearch/existing/readonly-repository\"," +
+                        "\"readonly\": \"true\"," +
+                        securityCredentials(securityEnabled, esKerberosPrincipal) +
+                        "\"conf.dfs.nameservices\": \"ha-hdfs\"," +
+                        "\"conf.dfs.ha.namenodes.ha-hdfs\": \"nn1,nn2\"," +
+                        "\"conf.dfs.namenode.rpc-address.ha-hdfs.nn1\": \"localhost:10001\"," +
+                        "\"conf.dfs.namenode.rpc-address.ha-hdfs.nn2\": \"localhost:10002\"," +
+                        "\"conf.dfs.client.failover.proxy.provider.ha-hdfs\": " +
+                            "\"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\"" +
+                    "}" +
+                "}",
+                Charset.defaultCharset()), contentHeader);
+
+            Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+        }
+
+        // Get repository
+        {
+            Response response = client.performRequest("GET", "/_snapshot/hdfs_ha_repo_read/_all", emptyParams);
+            Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+        }
+
+        // Failover the namenode to the second.
+        failoverHDFS("nn1", "nn2", hdfsConfiguration);
+
+        // Get repository again
+        {
+            Response response = client.performRequest("GET", "/_snapshot/hdfs_ha_repo_read/_all", emptyParams);
+            Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+        }
+    }
+
+    private String securityCredentials(boolean securityEnabled, String kerberosPrincipal) {
+        if (securityEnabled) {
+            return "\"security.principal\": \""+kerberosPrincipal+"\"," +
+                "\"conf.dfs.data.transfer.protection\": \"authentication\",";
+        } else {
+            return "";
+        }
+    }
+
+    /**
+     * Wraps an HAServiceTarget, keeping track of any HAServiceProtocol proxies it generates in order
+     * to close them at the end of the test lifecycle.
+     */
+    private static class CloseableHAServiceTarget extends HAServiceTarget {
+        private final HAServiceTarget delegate;
+        private final List protocolsToClose = new ArrayList<>();
+
+        CloseableHAServiceTarget(HAServiceTarget delegate) {
+            this.delegate = delegate;
+        }
+
+        @Override
+        public InetSocketAddress getAddress() {
+            return delegate.getAddress();
+        }
+
+        @Override
+        public InetSocketAddress getHealthMonitorAddress() {
+            return delegate.getHealthMonitorAddress();
+        }
+
+        @Override
+        public InetSocketAddress getZKFCAddress() {
+            return delegate.getZKFCAddress();
+        }
+
+        @Override
+        public NodeFencer getFencer() {
+            return delegate.getFencer();
+        }
+
+        @Override
+        public void checkFencingConfigured() throws BadFencingConfigurationException {
+            delegate.checkFencingConfigured();
+        }
+
+        @Override
+        public HAServiceProtocol getProxy(Configuration conf, int timeoutMs) throws IOException {
+            HAServiceProtocol proxy = delegate.getProxy(conf, timeoutMs);
+            protocolsToClose.add(proxy);
+            return proxy;
+        }
+
+        @Override
+        public HAServiceProtocol getHealthMonitorProxy(Configuration conf, int timeoutMs) throws IOException {
+            return delegate.getHealthMonitorProxy(conf, timeoutMs);
+        }
+
+        @Override
+        public ZKFCProtocol getZKFCProxy(Configuration conf, int timeoutMs) throws IOException {
+            return delegate.getZKFCProxy(conf, timeoutMs);
+        }
+
+        @Override
+        public boolean isAutoFailoverEnabled() {
+            return delegate.isAutoFailoverEnabled();
+        }
+
+        private void close() {
+            for (HAServiceProtocol protocol : protocolsToClose) {
+                if (protocol instanceof HAServiceProtocolClientSideTranslatorPB) {
+                    ((HAServiceProtocolClientSideTranslatorPB) protocol).close();
+                }
+            }
+        }
+    }
+
+    /**
+     * The default HAAdmin tool does not throw exceptions on failures, and does not close any client connection
+     * resources when it concludes. This subclass overrides the tool to allow for exception throwing, and to
+     * keep track of and clean up connection resources.
+     */
+    private static class CloseableHAAdmin extends DFSHAAdmin {
+        private final List serviceTargets = new ArrayList<>();
+
+        @Override
+        protected HAServiceTarget resolveTarget(String nnId) {
+            CloseableHAServiceTarget target = new CloseableHAServiceTarget(super.resolveTarget(nnId));
+            serviceTargets.add(target);
+            return target;
+        }
+
+        @Override
+        public int run(String[] argv) throws Exception {
+            return runCmd(argv);
+        }
+
+        public int transitionToStandby(String namenodeID) throws Exception {
+            return run(new String[]{"-transitionToStandby", namenodeID});
+        }
+
+        public int transitionToActive(String namenodeID) throws Exception {
+            return run(new String[]{"-transitionToActive", namenodeID});
+        }
+
+        public void close() {
+            for (CloseableHAServiceTarget serviceTarget : serviceTargets) {
+                serviceTarget.close();
+            }
+        }
+    }
+
+    /**
+     * Performs a two-phase leading namenode transition.
+     * @param from Namenode ID to transition to standby
+     * @param to Namenode ID to transition to active
+     * @param configuration Client configuration for HAAdmin tool
+     * @throws IOException In the event of a raised exception during namenode failover.
+     */
+    private void failoverHDFS(String from, String to, Configuration configuration) throws IOException {
+        logger.info("Swapping active namenodes: [{}] to standby and [{}] to active", from, to);
+        try {
+            AccessController.doPrivileged((PrivilegedExceptionAction) () -> {
+                CloseableHAAdmin haAdmin = new CloseableHAAdmin();
+                haAdmin.setConf(configuration);
+                try {
+                    haAdmin.transitionToStandby(from);
+                    haAdmin.transitionToActive(to);
+                } finally {
+                    haAdmin.close();
+                }
+                return null;
+            });
+        } catch (PrivilegedActionException pae) {
+            throw new IOException("Unable to perform namenode failover", pae);
+        }
+    }
+}
diff --git a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
index 73f4e443b07..0ddf7af6533 100644
--- a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
+++ b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
@@ -41,6 +41,9 @@ import org.apache.hadoop.fs.permission.AclEntryType;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 
 /**
@@ -91,6 +94,7 @@ public class MiniHDFS {
             cfg.set(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, "true");
             cfg.set(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, "true");
             cfg.set(DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY, "true");
+            cfg.set(DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY, "true");
         }
 
         UserGroupInformation.setConfiguration(cfg);
@@ -102,12 +106,32 @@ public class MiniHDFS {
         } else {
             builder.nameNodePort(9999);
         }
+
+        // Configure HA mode
+        String haNameService = System.getProperty("ha-nameservice");
+        boolean haEnabled = haNameService != null;
+        if (haEnabled) {
+            MiniDFSNNTopology.NNConf nn1 = new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10001);
+            MiniDFSNNTopology.NNConf nn2 = new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10002);
+            MiniDFSNNTopology.NSConf nameservice = new MiniDFSNNTopology.NSConf(haNameService).addNN(nn1).addNN(nn2);
+            MiniDFSNNTopology namenodeTopology = new MiniDFSNNTopology().addNameservice(nameservice);
+            builder.nnTopology(namenodeTopology);
+        }
+
         MiniDFSCluster dfs = builder.build();
 
         // Configure contents of the filesystem
         org.apache.hadoop.fs.Path esUserPath = new org.apache.hadoop.fs.Path("/user/elasticsearch");
-        try (FileSystem fs = dfs.getFileSystem()) {
 
+        FileSystem fs;
+        if (haEnabled) {
+            dfs.transitionToActive(0);
+            fs = HATestUtil.configureFailoverFs(dfs, cfg);
+        } else {
+            fs = dfs.getFileSystem();
+        }
+
+        try {
             // Set the elasticsearch user directory up
             fs.mkdirs(esUserPath);
             if (UserGroupInformation.isSecurityEnabled()) {
@@ -133,6 +157,8 @@ public class MiniHDFS {
 
                 FileUtils.deleteDirectory(tempDirectory.toFile());
             }
+        } finally {
+            fs.close();
         }
 
         // write our PID file
@@ -142,8 +168,12 @@ public class MiniHDFS {
         Files.move(tmp, baseDir.resolve(PID_FILE_NAME), StandardCopyOption.ATOMIC_MOVE);
 
         // write our port file
+        String portFileContent = Integer.toString(dfs.getNameNodePort(0));
+        if (haEnabled) {
+            portFileContent = portFileContent + "\n" + Integer.toString(dfs.getNameNodePort(1));
+        }
         tmp = Files.createTempFile(baseDir, null, null);
-        Files.write(tmp, Integer.toString(dfs.getNameNodePort()).getBytes(StandardCharsets.UTF_8));
+        Files.write(tmp, portFileContent.getBytes(StandardCharsets.UTF_8));
         Files.move(tmp, baseDir.resolve(PORT_FILE_NAME), StandardCopyOption.ATOMIC_MOVE);
     }
 }

From cd67f6a8d79daa2c6ed32bc0d4ca4b34c9659dc5 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Sun, 3 Dec 2017 08:33:21 -0500
Subject: [PATCH 150/297] Enable GC logs by default

For too long we have been groping around in the dark when faced with GC
issues because we rarely have GC logs at our disposal. This commit
enables GC logging by default out of the box.

Relates #27610
---
 distribution/build.gradle                     | 20 ++++++++++++++-
 distribution/integ-test-zip/build.gradle      |  2 +-
 .../src/main/resources/bin/elasticsearch      |  1 +
 .../src/main/resources/bin/elasticsearch.bat  |  1 +
 .../src/main/resources/config/jvm.options     | 25 ++++++-------------
 distribution/tar/build.gradle                 |  2 +-
 distribution/zip/build.gradle                 |  2 +-
 .../setup/important-settings.asciidoc         |  9 +++++++
 .../packaging/tests/20_tar_package.bats       |  6 +++++
 .../resources/packaging/tests/60_systemd.bats |  6 +++++
 .../packaging/tests/70_sysv_initd.bats        |  6 +++++
 .../test/resources/packaging/utils/tar.bash   |  1 +
 12 files changed, 60 insertions(+), 21 deletions(-)

diff --git a/distribution/build.gradle b/distribution/build.gradle
index 5c809de568d..b7fa48561da 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -224,6 +224,11 @@ configure(distributions) {
 configure(distributions.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.name) }) {
   // CopySpec does not make it easy to create an empty director so we create the directory that we want, and then point CopySpec to its
   // parent to copy to the root of the distribution
+  File logs = new File(buildDir, 'logs-hack/logs')
+  task createLogDir(type: EmptyDirTask) {
+    dir "${logs}"
+    dirMode 0755
+  }
   File plugins = new File(buildDir, 'plugins-hack/plugins')
   task createPluginsDir(type: EmptyDirTask) {
     dir "${plugins}"
@@ -247,6 +252,12 @@ configure(distributions.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.n
           MavenFilteringHack.filter(it, expansions)
         }
       }
+      into('') {
+        from {
+          dirMode 0755
+          logs.getParent()
+        }
+      }
       into('') {
         from {
           dirMode 0755
@@ -497,7 +508,9 @@ task run(type: RunTask) {
 Map expansionsForDistribution(distributionType) {
   final String defaultHeapSize = "1g"
   final String packagingPathData = "path.data: /var/lib/elasticsearch"
-  final String packagingPathLogs = "path.logs: /var/log/elasticsearch"
+  final String pathLogs = "/var/log/elasticsearch"
+  final String packagingPathLogs = "path.logs: ${pathLogs}"
+  final String packagingLoggc = "${pathLogs}/gc.log"
 
   String footer = "# Built for ${project.name}-${project.version} " +
       "(${distributionType})"
@@ -533,6 +546,11 @@ Map expansionsForDistribution(distributionType) {
       'rpm': packagingPathLogs,
       'def': '#path.logs: /path/to/logs'
     ],
+    'loggc': [
+      'deb': packagingLoggc,
+      'rpm': packagingLoggc,
+      'def': 'logs/gc.log'
+    ],
 
     'heap.min': defaultHeapSize,
     'heap.max': defaultHeapSize,
diff --git a/distribution/integ-test-zip/build.gradle b/distribution/integ-test-zip/build.gradle
index a99768f7cf9..89ad1ebee73 100644
--- a/distribution/integ-test-zip/build.gradle
+++ b/distribution/integ-test-zip/build.gradle
@@ -21,7 +21,7 @@ import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.apache.tools.ant.taskdefs.condition.Os
 
 task buildZip(type: Zip) {
-  dependsOn createPluginsDir
+  dependsOn createLogDir, createPluginsDir
   baseName = 'elasticsearch'
   with archivesFiles
 }
diff --git a/distribution/src/main/resources/bin/elasticsearch b/distribution/src/main/resources/bin/elasticsearch
index 6a63d2e9aa8..4064170807f 100755
--- a/distribution/src/main/resources/bin/elasticsearch
+++ b/distribution/src/main/resources/bin/elasticsearch
@@ -27,6 +27,7 @@ ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options
 JVM_OPTIONS=`parse_jvm_options "$ES_JVM_OPTIONS"`
 ES_JAVA_OPTS="${JVM_OPTIONS//\$\{ES_TMPDIR\}/$ES_TMPDIR} $ES_JAVA_OPTS"
 
+cd "$ES_HOME"
 # manual parsing to find out, if process should be detached
 if ! echo $* | grep -E '(^-d |-d$| -d |--daemonize$|--daemonize )' > /dev/null; then
   exec \
diff --git a/distribution/src/main/resources/bin/elasticsearch.bat b/distribution/src/main/resources/bin/elasticsearch.bat
index 5c172d20b82..210da3e5eb6 100644
--- a/distribution/src/main/resources/bin/elasticsearch.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.bat
@@ -49,6 +49,7 @@ rem such options are the lines beginning with '-', thus "findstr /b"
 for /F "usebackq delims=" %%a in (`findstr /b \- "%ES_JVM_OPTIONS%"`) do set JVM_OPTIONS=!JVM_OPTIONS! %%a
 @endlocal & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!% %ES_JAVA_OPTS%
 
+cd "%ES_HOME%"
 %JAVA% %ES_JAVA_OPTS% -Delasticsearch -Des.path.home="%ES_HOME%" -Des.path.conf="%ES_PATH_CONF%" -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams!
 
 endlocal
diff --git a/distribution/src/main/resources/config/jvm.options b/distribution/src/main/resources/config/jvm.options
index f5d527eec87..a8fff81f468 100644
--- a/distribution/src/main/resources/config/jvm.options
+++ b/distribution/src/main/resources/config/jvm.options
@@ -86,20 +86,11 @@ ${heap.dump.path}
 
 ## GC logging
 
-#-XX:+PrintGCDetails
-#-XX:+PrintGCTimeStamps
-#-XX:+PrintGCDateStamps
-#-XX:+PrintClassHistogram
-#-XX:+PrintTenuringDistribution
-#-XX:+PrintGCApplicationStoppedTime
-
-# log GC status to a file with time stamps
-# ensure the directory exists
-#-Xloggc:${loggc}
-
-# By default, the GC log file will not rotate.
-# By uncommenting the lines below, the GC log file
-# will be rotated every 128MB at most 32 times.
-#-XX:+UseGCLogFileRotation
-#-XX:NumberOfGCLogFiles=32
-#-XX:GCLogFileSize=128M
+-XX:+PrintGCDetails
+-XX:+PrintGCDateStamps
+-XX:+PrintTenuringDistribution
+-XX:+PrintGCApplicationStoppedTime
+-Xloggc:${loggc}
+-XX:+UseGCLogFileRotation
+-XX:NumberOfGCLogFiles=32
+-XX:GCLogFileSize=64m
diff --git a/distribution/tar/build.gradle b/distribution/tar/build.gradle
index 2140061ee48..cbefc223847 100644
--- a/distribution/tar/build.gradle
+++ b/distribution/tar/build.gradle
@@ -18,7 +18,7 @@
  */
 
 task buildTar(type: Tar) {
-  dependsOn createPluginsDir
+  dependsOn createLogDir, createPluginsDir
   baseName = 'elasticsearch'
   extension = 'tar.gz'
   with archivesFiles
diff --git a/distribution/zip/build.gradle b/distribution/zip/build.gradle
index 80da4131995..53dc98271ec 100644
--- a/distribution/zip/build.gradle
+++ b/distribution/zip/build.gradle
@@ -20,7 +20,7 @@
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 
 task buildZip(type: Zip) {
-  dependsOn createPluginsDir
+  dependsOn createLogDir, createPluginsDir
   baseName = 'elasticsearch'
   with archivesFiles
 }
diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc
index aa86e9be268..5186a8670e2 100644
--- a/docs/reference/setup/important-settings.asciidoc
+++ b/docs/reference/setup/important-settings.asciidoc
@@ -201,3 +201,12 @@ the Elasticsearch process. If you wish to configure a heap dump path, you should
 modify the entry `#-XX:HeapDumpPath=/heap/dump/path` in
 <> to remove the comment marker `#` and to specify an
 actual path.
+
+[float]
+[[gc-logging]]
+=== GC logging
+
+By default, Elasticsearch enables GC logs. These are configured in
+<> and default to the same default location as the
+Elasticsearch logs. The default configuration rotates the logs every 64 MB and
+can consume up to 2 GB of disk space.
diff --git a/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats b/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats
index 645c8331317..715d9958459 100644
--- a/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats
+++ b/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats
@@ -133,6 +133,12 @@ setup() {
     export ES_JAVA_OPTS=$es_java_opts
 }
 
+@test "[TAR] GC logs exist" {
+    start_elasticsearch_service
+    assert_file_exist $ESHOME/logs/gc.log.0.current
+    stop_elasticsearch_service
+}
+
 @test "[TAR] remove tar" {
     rm -rf "/tmp/elasticsearch"
 }
diff --git a/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats b/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats
index c57afb45653..cb9e6658d3d 100644
--- a/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats
+++ b/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats
@@ -246,3 +246,9 @@ setup() {
     [ -d /var/run/elasticsearch ]
     systemctl stop elasticsearch.service
 }
+
+@test "[SYSTEMD] GC logs exist" {
+    start_elasticsearch_service
+    assert_file_exist /var/log/elasticsearch/gc.log.0.current
+    stop_elasticsearch_service
+}
diff --git a/qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats b/qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats
index 63186e63fb5..026b46e21bc 100644
--- a/qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats
+++ b/qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats
@@ -157,3 +157,9 @@ setup() {
 
     service elasticsearch stop
 }
+
+@test "[INIT.D] GC logs exist" {
+    start_elasticsearch_service
+    assert_file_exist /var/log/elasticsearch/gc.log.0.current
+    stop_elasticsearch_service
+}
diff --git a/qa/vagrant/src/test/resources/packaging/utils/tar.bash b/qa/vagrant/src/test/resources/packaging/utils/tar.bash
index b56c372d516..d96d360a02b 100644
--- a/qa/vagrant/src/test/resources/packaging/utils/tar.bash
+++ b/qa/vagrant/src/test/resources/packaging/utils/tar.bash
@@ -97,6 +97,7 @@ verify_archive_installation() {
     assert_file "$ESCONFIG/log4j2.properties" f elasticsearch elasticsearch 660
     assert_file "$ESPLUGINS" d elasticsearch elasticsearch 755
     assert_file "$ESHOME/lib" d elasticsearch elasticsearch 755
+    assert_file "$ESHOME/logs" d elasticsearch elasticsearch 755
     assert_file "$ESHOME/NOTICE.txt" f elasticsearch elasticsearch 644
     assert_file "$ESHOME/LICENSE.txt" f elasticsearch elasticsearch 644
     assert_file "$ESHOME/README.textile" f elasticsearch elasticsearch 644

From a880bbd57d3a0e4d3c992bb7f28ea2a7b91f1e98 Mon Sep 17 00:00:00 2001
From: ajrpayne 
Date: Sun, 3 Dec 2017 06:20:10 -0800
Subject: [PATCH 151/297] Reflect changes in systemd service for LimitMEMLOCK

The LimitMEMLOCK suggestion was removed from systemd service file and
instead users should use an override file, so a comment in the
environment file should be updated to reflect the same.

Relates #27630
---
 distribution/src/main/packaging/env/elasticsearch | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distribution/src/main/packaging/env/elasticsearch b/distribution/src/main/packaging/env/elasticsearch
index 1c409535137..995a264e513 100644
--- a/distribution/src/main/packaging/env/elasticsearch
+++ b/distribution/src/main/packaging/env/elasticsearch
@@ -41,8 +41,8 @@ ES_STARTUP_SLEEP_TIME=5
 # The maximum number of bytes of memory that may be locked into RAM
 # Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option
 # in elasticsearch.yml.
-# When using Systemd, the LimitMEMLOCK property must be set
-# in /usr/lib/systemd/system/elasticsearch.service
+# When using systemd, LimitMEMLOCK must be set in a unit file such as
+# /etc/systemd/system/elasticsearch.service.d/override.conf.
 #MAX_LOCKED_MEMORY=unlimited
 
 # Maximum number of VMA (Virtual Memory Areas) a process can own

From 49df50f6621477322cbed3694ae7416283ef1400 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen 
Date: Sun, 3 Dec 2017 15:20:57 -0500
Subject: [PATCH 152/297] Simplify MultiSnapshot#SeqNoset (#27547)

Today, we maintain two sets in a SeqNoSet: ongoing sets and completed
sets. We can remove the completed sets and use only the ongoing sets by
releasing the internal bitset of a CountedBitSet when all its bits are
set. This behaves like two sets but simpler. This commit also makes
CountedBitSet as a drop-in replacement for BitSet.

Relates #27268
---
 .../index/translog/CountedBitSet.java         | 106 ++++++++++++++++++
 .../index/translog/MultiSnapshot.java         |  66 ++---------
 .../index/translog/CountedBitSetTests.java    |  97 ++++++++++++++++
 .../index/translog/MultiSnapshotTests.java    |  16 ---
 4 files changed, 210 insertions(+), 75 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/index/translog/CountedBitSet.java
 create mode 100644 core/src/test/java/org/elasticsearch/index/translog/CountedBitSetTests.java

diff --git a/core/src/main/java/org/elasticsearch/index/translog/CountedBitSet.java b/core/src/main/java/org/elasticsearch/index/translog/CountedBitSet.java
new file mode 100644
index 00000000000..9fac230c9a8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/translog/CountedBitSet.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.translog;
+
+import org.apache.lucene.util.BitSet;
+import org.apache.lucene.util.FixedBitSet;
+
+/**
+ * A {@link CountedBitSet} wraps a {@link FixedBitSet} but automatically releases the internal bitset
+ * when all bits are set to reduce memory usage. This structure can work well for sequence numbers
+ * from translog as these numbers are likely to form contiguous ranges (eg. filling all bits).
+ */
+final class CountedBitSet extends BitSet {
+    private short onBits; // Number of bits are set.
+    private FixedBitSet bitset;
+
+    CountedBitSet(short numBits) {
+        assert numBits > 0;
+        this.onBits = 0;
+        this.bitset = new FixedBitSet(numBits);
+    }
+
+    @Override
+    public boolean get(int index) {
+        assert 0 <= index && index < this.length();
+        assert bitset == null || onBits < bitset.length() : "Bitset should be released when all bits are set";
+
+        return bitset == null ? true : bitset.get(index);
+    }
+
+    @Override
+    public void set(int index) {
+        assert 0 <= index && index < this.length();
+        assert bitset == null || onBits < bitset.length() : "Bitset should be released when all bits are set";
+
+        // Ignore set when bitset is full.
+        if (bitset != null) {
+            boolean wasOn = bitset.getAndSet(index);
+            if (wasOn == false) {
+                onBits++;
+                // Once all bits are set, we can simply just return YES for all indexes.
+                // This allows us to clear the internal bitset and use null check as the guard.
+                if (onBits == bitset.length()) {
+                    bitset = null;
+                }
+            }
+        }
+    }
+
+    @Override
+    public void clear(int startIndex, int endIndex) {
+        throw new UnsupportedOperationException("Not implemented yet");
+    }
+
+    @Override
+    public void clear(int index) {
+        throw new UnsupportedOperationException("Not implemented yet");
+    }
+
+    @Override
+    public int cardinality() {
+        return onBits;
+    }
+
+    @Override
+    public int length() {
+        return bitset == null ? onBits : bitset.length();
+    }
+
+    @Override
+    public int prevSetBit(int index) {
+        throw new UnsupportedOperationException("Not implemented yet");
+    }
+
+    @Override
+    public int nextSetBit(int index) {
+        throw new UnsupportedOperationException("Not implemented yet");
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        throw new UnsupportedOperationException("Not implemented yet");
+    }
+
+    // Exposed for testing
+    boolean isInternalBitsetReleased() {
+        return bitset == null;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java b/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java
index cc9dbdeb63f..e4bfbdcd42e 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java
@@ -19,10 +19,8 @@
 
 package org.elasticsearch.index.translog;
 
-import com.carrotsearch.hppc.LongHashSet;
 import com.carrotsearch.hppc.LongObjectHashMap;
-import com.carrotsearch.hppc.LongSet;
-import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.BitSet;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 
 import java.io.Closeable;
@@ -84,41 +82,9 @@ final class MultiSnapshot implements Translog.Snapshot {
         onClose.close();
     }
 
-    /**
-     * A wrapper of {@link FixedBitSet} but allows to check if all bits are set in O(1).
-     */
-    private static final class CountedBitSet {
-        private short onBits;
-        private final FixedBitSet bitset;
-
-        CountedBitSet(short numBits) {
-            assert numBits > 0;
-            this.onBits = 0;
-            this.bitset = new FixedBitSet(numBits);
-        }
-
-        boolean getAndSet(int index) {
-            assert index >= 0;
-            boolean wasOn = bitset.getAndSet(index);
-            if (wasOn == false) {
-                onBits++;
-            }
-            return wasOn;
-        }
-
-        boolean hasAllBitsOn() {
-            return onBits == bitset.length();
-        }
-    }
-
-    /**
-     * Sequence numbers from translog are likely to form contiguous ranges,
-     * thus collapsing a completed bitset into a single entry will reduce memory usage.
-     */
     static final class SeqNoSet {
         static final short BIT_SET_SIZE = 1024;
-        private final LongSet completedSets = new LongHashSet();
-        private final LongObjectHashMap ongoingSets = new LongObjectHashMap<>();
+        private final LongObjectHashMap bitSets = new LongObjectHashMap<>();
 
         /**
          * Marks this sequence number and returns true if it is seen before.
@@ -126,33 +92,15 @@ final class MultiSnapshot implements Translog.Snapshot {
         boolean getAndSet(long value) {
             assert value >= 0;
             final long key = value / BIT_SET_SIZE;
-
-            if (completedSets.contains(key)) {
-                return true;
-            }
-
-            CountedBitSet bitset = ongoingSets.get(key);
+            BitSet bitset = bitSets.get(key);
             if (bitset == null) {
                 bitset = new CountedBitSet(BIT_SET_SIZE);
-                ongoingSets.put(key, bitset);
-            }
-
-            final boolean wasOn = bitset.getAndSet(Math.toIntExact(value % BIT_SET_SIZE));
-            if (bitset.hasAllBitsOn()) {
-                ongoingSets.remove(key);
-                completedSets.add(key);
+                bitSets.put(key, bitset);
             }
+            final int index = Math.toIntExact(value % BIT_SET_SIZE);
+            final boolean wasOn = bitset.get(index);
+            bitset.set(index);
             return wasOn;
         }
-
-        // For testing
-        long completeSetsSize() {
-            return completedSets.size();
-        }
-
-        // For testing
-        long ongoingSetsSize() {
-            return ongoingSets.size();
-        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/translog/CountedBitSetTests.java b/core/src/test/java/org/elasticsearch/index/translog/CountedBitSetTests.java
new file mode 100644
index 00000000000..5174d1755be
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/translog/CountedBitSetTests.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.translog;
+
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CountedBitSetTests extends ESTestCase {
+
+    public void testCompareToFixedBitset() {
+        int numBits = (short) randomIntBetween(8, 4096);
+        final FixedBitSet fixedBitSet = new FixedBitSet(numBits);
+        final CountedBitSet countedBitSet = new CountedBitSet((short) numBits);
+
+        for (int i = 0; i < numBits; i++) {
+            if (randomBoolean()) {
+                fixedBitSet.set(i);
+                countedBitSet.set(i);
+            }
+            assertThat(countedBitSet.cardinality(), equalTo(fixedBitSet.cardinality()));
+            assertThat(countedBitSet.length(), equalTo(fixedBitSet.length()));
+        }
+
+        for (int i = 0; i < numBits; i++) {
+            assertThat(countedBitSet.get(i), equalTo(fixedBitSet.get(i)));
+        }
+    }
+
+    public void testReleaseInternalBitSet() {
+        int numBits = (short) randomIntBetween(8, 4096);
+        final CountedBitSet countedBitSet = new CountedBitSet((short) numBits);
+        final List values = IntStream.range(0, numBits).boxed().collect(Collectors.toList());
+
+        for (int i = 1; i < numBits; i++) {
+            final int value = values.get(i);
+            assertThat(countedBitSet.get(value), equalTo(false));
+            assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(false));
+
+            countedBitSet.set(value);
+
+            assertThat(countedBitSet.get(value), equalTo(true));
+            assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(false));
+            assertThat(countedBitSet.length(), equalTo(numBits));
+            assertThat(countedBitSet.cardinality(), equalTo(i));
+        }
+
+        // The missing piece to fill all bits.
+        {
+            final int value = values.get(0);
+            assertThat(countedBitSet.get(value), equalTo(false));
+            assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(false));
+
+            countedBitSet.set(value);
+
+            assertThat(countedBitSet.get(value), equalTo(true));
+            assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(true));
+            assertThat(countedBitSet.length(), equalTo(numBits));
+            assertThat(countedBitSet.cardinality(), equalTo(numBits));
+        }
+
+        // Tests with released internal bitset.
+        final int iterations = iterations(1000, 10000);
+        for (int i = 0; i < iterations; i++) {
+            final int value = randomInt(numBits - 1);
+            assertThat(countedBitSet.get(value), equalTo(true));
+            assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(true));
+            assertThat(countedBitSet.length(), equalTo(numBits));
+            assertThat(countedBitSet.cardinality(), equalTo(numBits));
+            if (frequently()) {
+                assertThat(countedBitSet.get(value), equalTo(true));
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/translog/MultiSnapshotTests.java b/core/src/test/java/org/elasticsearch/index/translog/MultiSnapshotTests.java
index 7ee2a6c3366..31dda2cb921 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/MultiSnapshotTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/MultiSnapshotTests.java
@@ -30,7 +30,6 @@ import java.util.stream.IntStream;
 import java.util.stream.LongStream;
 
 import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class MultiSnapshotTests extends ESTestCase {
 
@@ -40,14 +39,8 @@ public class MultiSnapshotTests extends ESTestCase {
         Randomness.shuffle(values);
         for (int i = 0; i < 1023; i++) {
             assertThat(bitSet.getAndSet(values.get(i)), equalTo(false));
-            assertThat(bitSet.ongoingSetsSize(), equalTo(1L));
-            assertThat(bitSet.completeSetsSize(), equalTo(0L));
         }
-
         assertThat(bitSet.getAndSet(values.get(1023)), equalTo(false));
-        assertThat(bitSet.ongoingSetsSize(), equalTo(0L));
-        assertThat(bitSet.completeSetsSize(), equalTo(1L));
-
         assertThat(bitSet.getAndSet(between(0, 1023)), equalTo(true));
         assertThat(bitSet.getAndSet(between(1024, Integer.MAX_VALUE)), equalTo(false));
     }
@@ -59,7 +52,6 @@ public class MultiSnapshotTests extends ESTestCase {
             long seq = between(0, 5000);
             boolean existed = normalSet.add(seq) == false;
             assertThat("SeqNoSet != Set" + seq, bitSet.getAndSet(seq), equalTo(existed));
-            assertThat(bitSet.ongoingSetsSize() + bitSet.completeSetsSize(), lessThanOrEqualTo(5L));
         });
     }
 
@@ -78,12 +70,8 @@ public class MultiSnapshotTests extends ESTestCase {
         final LongSet normalSet = new LongHashSet();
         long currentSeq = between(10_000_000, 1_000_000_000);
         final int iterations = scaledRandomIntBetween(100, 2000);
-        assertThat(bitSet.completeSetsSize(), equalTo(0L));
-        assertThat(bitSet.ongoingSetsSize(), equalTo(0L));
-        long totalDocs = 0;
         for (long i = 0; i < iterations; i++) {
             int batchSize = between(1, 1500);
-            totalDocs += batchSize;
             currentSeq -= batchSize;
             List batch = LongStream.range(currentSeq, currentSeq + batchSize)
                 .boxed()
@@ -92,11 +80,7 @@ public class MultiSnapshotTests extends ESTestCase {
             batch.forEach(seq -> {
                 boolean existed = normalSet.add(seq) == false;
                 assertThat("SeqNoSet != Set", bitSet.getAndSet(seq), equalTo(existed));
-                assertThat(bitSet.ongoingSetsSize(), lessThanOrEqualTo(4L));
             });
-            assertThat(bitSet.ongoingSetsSize(), lessThanOrEqualTo(2L));
         }
-        assertThat(bitSet.completeSetsSize(), lessThanOrEqualTo(totalDocs / 1024));
-        assertThat(bitSet.ongoingSetsSize(), lessThanOrEqualTo(2L));
     }
 }

From 6323bb0d978cf448f2f12db6916d3ef2d9ee38a4 Mon Sep 17 00:00:00 2001
From: Adrien Grand 
Date: Mon, 4 Dec 2017 09:40:08 +0100
Subject: [PATCH 153/297] Upgrade to lucene-7.2.0-snapshot-8c94404. (#27619)

This new snapshot mostly brings a change to TopFieldCollector which can now
early terminate collection when trackTotalHits is `false`.

As a follow-up, we should replace our usage of
`EarlyTerminatingSortingCollector` with this new option.
---
 buildSrc/version.properties                                   | 2 +-
 .../lucene-analyzers-common-7.2.0-snapshot-7deca62.jar.sha1   | 1 +
 .../lucene-analyzers-common-7.2.0-snapshot-8c94404.jar.sha1   | 1 -
 .../lucene-backward-codecs-7.2.0-snapshot-7deca62.jar.sha1    | 1 +
 .../lucene-backward-codecs-7.2.0-snapshot-8c94404.jar.sha1    | 1 -
 core/licenses/lucene-core-7.2.0-snapshot-7deca62.jar.sha1     | 1 +
 core/licenses/lucene-core-7.2.0-snapshot-8c94404.jar.sha1     | 1 -
 core/licenses/lucene-grouping-7.2.0-snapshot-7deca62.jar.sha1 | 1 +
 core/licenses/lucene-grouping-7.2.0-snapshot-8c94404.jar.sha1 | 1 -
 .../lucene-highlighter-7.2.0-snapshot-7deca62.jar.sha1        | 1 +
 .../lucene-highlighter-7.2.0-snapshot-8c94404.jar.sha1        | 1 -
 core/licenses/lucene-join-7.2.0-snapshot-7deca62.jar.sha1     | 1 +
 core/licenses/lucene-join-7.2.0-snapshot-8c94404.jar.sha1     | 1 -
 core/licenses/lucene-memory-7.2.0-snapshot-7deca62.jar.sha1   | 1 +
 core/licenses/lucene-memory-7.2.0-snapshot-8c94404.jar.sha1   | 1 -
 core/licenses/lucene-misc-7.2.0-snapshot-7deca62.jar.sha1     | 1 +
 core/licenses/lucene-misc-7.2.0-snapshot-8c94404.jar.sha1     | 1 -
 core/licenses/lucene-queries-7.2.0-snapshot-7deca62.jar.sha1  | 1 +
 core/licenses/lucene-queries-7.2.0-snapshot-8c94404.jar.sha1  | 1 -
 .../lucene-queryparser-7.2.0-snapshot-7deca62.jar.sha1        | 1 +
 .../lucene-queryparser-7.2.0-snapshot-8c94404.jar.sha1        | 1 -
 core/licenses/lucene-sandbox-7.2.0-snapshot-7deca62.jar.sha1  | 1 +
 core/licenses/lucene-sandbox-7.2.0-snapshot-8c94404.jar.sha1  | 1 -
 core/licenses/lucene-spatial-7.2.0-snapshot-7deca62.jar.sha1  | 1 +
 core/licenses/lucene-spatial-7.2.0-snapshot-8c94404.jar.sha1  | 1 -
 .../lucene-spatial-extras-7.2.0-snapshot-7deca62.jar.sha1     | 1 +
 .../lucene-spatial-extras-7.2.0-snapshot-8c94404.jar.sha1     | 1 -
 .../licenses/lucene-spatial3d-7.2.0-snapshot-7deca62.jar.sha1 | 1 +
 .../licenses/lucene-spatial3d-7.2.0-snapshot-8c94404.jar.sha1 | 1 -
 core/licenses/lucene-suggest-7.2.0-snapshot-7deca62.jar.sha1  | 1 +
 core/licenses/lucene-suggest-7.2.0-snapshot-8c94404.jar.sha1  | 1 -
 .../org/elasticsearch/index/query/NestedQueryBuilder.java     | 2 +-
 .../aggregations/metrics/tophits/TopHitsAggregator.java       | 4 +++-
 .../elasticsearch/search/query/TopDocsCollectorContext.java   | 2 +-
 .../lucene/grouping/CollapsingTopDocsCollectorTests.java      | 2 +-
 .../lucene-expressions-7.2.0-snapshot-7deca62.jar.sha1        | 1 +
 .../lucene-expressions-7.2.0-snapshot-8c94404.jar.sha1        | 1 -
 .../join/query/ParentChildInnerHitContextBuilder.java         | 4 ++--
 .../lucene-analyzers-icu-7.2.0-snapshot-7deca62.jar.sha1      | 1 +
 .../lucene-analyzers-icu-7.2.0-snapshot-8c94404.jar.sha1      | 1 -
 .../lucene-analyzers-kuromoji-7.2.0-snapshot-7deca62.jar.sha1 | 1 +
 .../lucene-analyzers-kuromoji-7.2.0-snapshot-8c94404.jar.sha1 | 1 -
 .../lucene-analyzers-phonetic-7.2.0-snapshot-7deca62.jar.sha1 | 1 +
 .../lucene-analyzers-phonetic-7.2.0-snapshot-8c94404.jar.sha1 | 1 -
 .../lucene-analyzers-smartcn-7.2.0-snapshot-7deca62.jar.sha1  | 1 +
 .../lucene-analyzers-smartcn-7.2.0-snapshot-8c94404.jar.sha1  | 1 -
 .../lucene-analyzers-stempel-7.2.0-snapshot-7deca62.jar.sha1  | 1 +
 .../lucene-analyzers-stempel-7.2.0-snapshot-8c94404.jar.sha1  | 1 -
 ...ucene-analyzers-morfologik-7.2.0-snapshot-7deca62.jar.sha1 | 1 +
 ...ucene-analyzers-morfologik-7.2.0-snapshot-8c94404.jar.sha1 | 1 -
 50 files changed, 31 insertions(+), 29 deletions(-)
 create mode 100644 core/licenses/lucene-analyzers-common-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-analyzers-common-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-backward-codecs-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-backward-codecs-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-core-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-core-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-grouping-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-grouping-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-highlighter-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-highlighter-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-join-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-join-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-memory-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-memory-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-misc-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-misc-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-queries-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-queries-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-queryparser-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-queryparser-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-sandbox-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-sandbox-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-spatial-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-spatial-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-spatial-extras-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-spatial-extras-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-spatial3d-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-spatial3d-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 core/licenses/lucene-suggest-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 core/licenses/lucene-suggest-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 modules/lang-expression/licenses/lucene-expressions-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.2.0-snapshot-8c94404.jar.sha1
 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.2.0-snapshot-7deca62.jar.sha1
 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.2.0-snapshot-8c94404.jar.sha1

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index d1d04ea27b5..dd66a291814 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.2.0-snapshot-8c94404
+lucene            = 7.2.0-snapshot-7deca62
 
 # optional dependencies
 spatial4j         = 0.6
diff --git a/core/licenses/lucene-analyzers-common-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-analyzers-common-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..cd7bfdd92d0
--- /dev/null
+++ b/core/licenses/lucene-analyzers-common-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+0078019336ebd3ee475019c88a749ce8b7b261fd
\ No newline at end of file
diff --git a/core/licenses/lucene-analyzers-common-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-analyzers-common-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 36a620fef18..00000000000
--- a/core/licenses/lucene-analyzers-common-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4c515e5152e6938129a5e97c5afb5b3b360faed3
\ No newline at end of file
diff --git a/core/licenses/lucene-backward-codecs-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-backward-codecs-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..bda523e8e7e
--- /dev/null
+++ b/core/licenses/lucene-backward-codecs-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+e66dfb2c0fb938a41d27059c923c3163187584c9
\ No newline at end of file
diff --git a/core/licenses/lucene-backward-codecs-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-backward-codecs-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 5616175485e..00000000000
--- a/core/licenses/lucene-backward-codecs-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-406c6cc0f8c2a47d42a1e343eaf2ad939fee905c
\ No newline at end of file
diff --git a/core/licenses/lucene-core-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-core-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..46a52029716
--- /dev/null
+++ b/core/licenses/lucene-core-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+b79166ddbca0fddcb794026a22c363be1caecf5d
\ No newline at end of file
diff --git a/core/licenses/lucene-core-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-core-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index c0abd20b083..00000000000
--- a/core/licenses/lucene-core-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4c93f7fbc6e0caf87f7948b8481d80e0167133bf
\ No newline at end of file
diff --git a/core/licenses/lucene-grouping-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-grouping-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..21c96b5a27c
--- /dev/null
+++ b/core/licenses/lucene-grouping-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+dacc60bd0f55f75b24c434e2d715a10a91127fd0
\ No newline at end of file
diff --git a/core/licenses/lucene-grouping-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-grouping-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 823ae2c5b48..00000000000
--- a/core/licenses/lucene-grouping-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b078ca50c6d579085c7755b4fd8de60711964dcc
\ No newline at end of file
diff --git a/core/licenses/lucene-highlighter-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-highlighter-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..acb9b3a4171
--- /dev/null
+++ b/core/licenses/lucene-highlighter-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+9debc384c54d36b69aa8cd990fd047fc9ef00eea
\ No newline at end of file
diff --git a/core/licenses/lucene-highlighter-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-highlighter-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 9f936b25729..00000000000
--- a/core/licenses/lucene-highlighter-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fc5e61c8879f22b65ee053f1665bc9f13af79c1d
\ No newline at end of file
diff --git a/core/licenses/lucene-join-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-join-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..644732a9fa2
--- /dev/null
+++ b/core/licenses/lucene-join-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+e048857ea7c66579d172295fc7394aa6163273d7
\ No newline at end of file
diff --git a/core/licenses/lucene-join-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-join-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 9bb132317db..00000000000
--- a/core/licenses/lucene-join-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9a10839d3dfe7b369f0af8a78a630ee4d82e678e
\ No newline at end of file
diff --git a/core/licenses/lucene-memory-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-memory-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..b493ef8be62
--- /dev/null
+++ b/core/licenses/lucene-memory-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+9e8da627bcd3934e8b921810e71f225b1d2312ec
\ No newline at end of file
diff --git a/core/licenses/lucene-memory-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-memory-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index f7f827c85d1..00000000000
--- a/core/licenses/lucene-memory-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d45f2f51cf6f47a66ecafddecb83c1e08eb4061f
\ No newline at end of file
diff --git a/core/licenses/lucene-misc-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-misc-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..ad204b217a9
--- /dev/null
+++ b/core/licenses/lucene-misc-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+dfb4feab274c56d8a45098a398ea623e0d9d2a41
\ No newline at end of file
diff --git a/core/licenses/lucene-misc-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-misc-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index fde0edc0d99..00000000000
--- a/core/licenses/lucene-misc-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-19cb7362be57104ad891259060af80fb4679e92c
\ No newline at end of file
diff --git a/core/licenses/lucene-queries-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-queries-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..c08b0098e40
--- /dev/null
+++ b/core/licenses/lucene-queries-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+e7962cf1c6a9654d428e0dceead0278083429044
\ No newline at end of file
diff --git a/core/licenses/lucene-queries-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-queries-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index ef6408913d2..00000000000
--- a/core/licenses/lucene-queries-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ae24737048d95f56d0099fea77498324412eef50
\ No newline at end of file
diff --git a/core/licenses/lucene-queryparser-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-queryparser-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..b3b7fb202af
--- /dev/null
+++ b/core/licenses/lucene-queryparser-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+6cc9de8bd74ded29572e0ca7ce14dc0ef11cefc4
\ No newline at end of file
diff --git a/core/licenses/lucene-queryparser-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-queryparser-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 1670bad8f3f..00000000000
--- a/core/licenses/lucene-queryparser-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a9d3422c9a72106026c19a8f76a4f4e62159ff5c
\ No newline at end of file
diff --git a/core/licenses/lucene-sandbox-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-sandbox-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..91a3f712a8a
--- /dev/null
+++ b/core/licenses/lucene-sandbox-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+69f59c8ee5b0cad345af52f7ef1901b5828cf41a
\ No newline at end of file
diff --git a/core/licenses/lucene-sandbox-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-sandbox-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index c8665f0c485..00000000000
--- a/core/licenses/lucene-sandbox-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-66433006587ede3e01899fd6f5e55c8378032c2f
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-spatial-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..aab3565a34c
--- /dev/null
+++ b/core/licenses/lucene-spatial-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+de04b5de5f08ef4699cbf3415753661ac69a1fda
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-spatial-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 7254970749f..00000000000
--- a/core/licenses/lucene-spatial-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b6b3082ba845f7bd41641b015624f46d4f20afb6
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial-extras-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-spatial-extras-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..faeaa4eabe3
--- /dev/null
+++ b/core/licenses/lucene-spatial-extras-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+92c74817922c9ed6ec484453e21ffc5ba802b56c
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial-extras-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-spatial-extras-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index bb62d7b5997..00000000000
--- a/core/licenses/lucene-spatial-extras-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7757cac49cb3e9f1a219346ce95fb80f61f7090e
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial3d-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-spatial3d-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..4b0d4b68751
--- /dev/null
+++ b/core/licenses/lucene-spatial3d-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+99eff7d10daf8d53bba5a173a833dd2ac83fca55
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial3d-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-spatial3d-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 78d8c2f5baf..00000000000
--- a/core/licenses/lucene-spatial3d-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-92991fdcd185883050d9530ccc0d863b7a08e99c
\ No newline at end of file
diff --git a/core/licenses/lucene-suggest-7.2.0-snapshot-7deca62.jar.sha1 b/core/licenses/lucene-suggest-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..567dc0c4163
--- /dev/null
+++ b/core/licenses/lucene-suggest-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+1d44a9ed8ecefd3c6dceffa99f896ba957b7c31e
\ No newline at end of file
diff --git a/core/licenses/lucene-suggest-7.2.0-snapshot-8c94404.jar.sha1 b/core/licenses/lucene-suggest-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 5c13208494c..00000000000
--- a/core/licenses/lucene-suggest-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fb6a94b833a23a17e3721ea2f9679ad770dec48b
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index ccf5ad71078..95e9e3a1869 100644
--- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -399,7 +399,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder
                     int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc());
                     TopDocsCollector topDocsCollector;
                     if (sort() != null) {
-                        topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
+                        topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores(), true);
                     } else {
                         topDocsCollector = TopScoreDocCollector.create(topN);
                     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
index 700acdf797a..e59299754ae 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
@@ -117,8 +117,10 @@ public class TopHitsAggregator extends MetricsAggregator {
                     if (sort == null) {
                         topDocsCollector = TopScoreDocCollector.create(topN);
                     } else {
+                        // TODO: can we pass trackTotalHits=subSearchContext.trackTotalHits(){
+                        // Note that this would require to catch CollectionTerminatedException
                         topDocsCollector = TopFieldCollector.create(sort.sort, topN, true, subSearchContext.trackScores(),
-                            subSearchContext.trackScores());
+                            subSearchContext.trackScores(), true);
                     }
                     topDocsCollectors.put(bucket, topDocsCollector);
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java b/core/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java
index 0c14ef6a815..a5eff585847 100644
--- a/core/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java
+++ b/core/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java
@@ -175,7 +175,7 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext {
                 this.topDocsCollector = TopScoreDocCollector.create(numHits, searchAfter);
             } else {
                 this.topDocsCollector = TopFieldCollector.create(sortAndFormats.sort, numHits,
-                    (FieldDoc) searchAfter, true, trackMaxScore, trackMaxScore);
+                    (FieldDoc) searchAfter, true, trackMaxScore, trackMaxScore, true);
             }
         }
 
diff --git a/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java b/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java
index 4352f16c05f..0290a6c5d10 100644
--- a/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java
+++ b/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java
@@ -130,7 +130,7 @@ public class CollapsingTopDocsCollectorTests extends ESTestCase {
         }
 
         TopFieldCollector topFieldCollector =
-            TopFieldCollector.create(sort, totalHits, true, trackMaxScores, trackMaxScores);
+            TopFieldCollector.create(sort, totalHits, true, trackMaxScores, trackMaxScores, true);
 
         searcher.search(new MatchAllDocsQuery(), collapsingCollector);
         searcher.search(new MatchAllDocsQuery(), topFieldCollector);
diff --git a/modules/lang-expression/licenses/lucene-expressions-7.2.0-snapshot-7deca62.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..0cce9fadc5c
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+fc180cf3b83df42ebdc18416ddbca2c4890914c3
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-7.2.0-snapshot-8c94404.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 26ac38e7a0a..00000000000
--- a/modules/lang-expression/licenses/lucene-expressions-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5d4b0551a5f745ddf630184b8f63d9d03b4f4003
\ No newline at end of file
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
index dd1009d7752..b2c09b5cbdb 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java
@@ -170,7 +170,7 @@ class ParentChildInnerHitContextBuilder extends InnerHitContextBuilder {
                     int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc());
                     TopDocsCollector topDocsCollector;
                     if (sort() != null) {
-                        topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
+                        topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores(), true);
                     } else {
                         topDocsCollector = TopScoreDocCollector.create(topN);
                     }
@@ -261,7 +261,7 @@ class ParentChildInnerHitContextBuilder extends InnerHitContextBuilder {
                     int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc());
                     TopDocsCollector topDocsCollector;
                     if (sort() != null) {
-                        topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
+                        topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores(), true);
                     } else {
                         topDocsCollector = TopScoreDocCollector.create(topN);
                     }
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.2.0-snapshot-7deca62.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..702b410166c
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+28f92af20fc24d2adba2cab91ca5d7dfcf6ddf34
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index e0cbc64c1a1..00000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f18454bf4be4698f7e6f3a7c950f75ee3fa4436e
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.2.0-snapshot-7deca62.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..20e2c0e2517
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+85eb5c6a3d27be95bd7a982055e20dcacb242a6b
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index fd6989ffad8..00000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a09ec54a9434987a16a4d9b68ca301a5a3bf4123
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.2.0-snapshot-7deca62.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..d16638160f1
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+b4e2d20a4ec764b73f7a64200bfe4de8c7928c9e
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 56f78f71f6c..00000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f86560f04296d8d34fed535abbf8002e0985b1df
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.2.0-snapshot-7deca62.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..9a9f39d130c
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+df3bd90eab8edc2ad1ee9212f9589189f0007653
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index a7abf2370cc..00000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dec7d0bee4de2ac8eb5aec9cb5d244c5b405bc15
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.2.0-snapshot-7deca62.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..e4031e9d224
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+bef9b75d220eb4f95a91696c69ae6f5b489b4a43
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index 67278daf3fc..00000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8469b05416dcaf6ddeebcdabb551b4ccf009f7c2
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.2.0-snapshot-7deca62.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.2.0-snapshot-7deca62.jar.sha1
new file mode 100644
index 00000000000..07a8493d097
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.2.0-snapshot-7deca62.jar.sha1
@@ -0,0 +1 @@
+ae0c475cfbc5d9a9eec79daff9260f1bbf12daed
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.2.0-snapshot-8c94404.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.2.0-snapshot-8c94404.jar.sha1
deleted file mode 100644
index fd0478fd9f5..00000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.2.0-snapshot-8c94404.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6de67186607bde2ac66b216af4e1fadffaf43e21
\ No newline at end of file

From c4fe7d3f7248223d5174b36fd4e1678217a6a6ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?=
 
Date: Mon, 27 Nov 2017 13:02:23 +0100
Subject: [PATCH 154/297] [Docs] add deprecation warning for
 `delimited_payload_filter` renaming

---
 .../tokenfilters/delimited-payload-tokenfilter.asciidoc    | 7 +++++++
 .../common/LegacyDelimitedPayloadTokenFilterFactory.java   | 2 +-
 .../test/analysis-common/40_token_filters.yml              | 4 ++--
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/docs/reference/analysis/tokenfilters/delimited-payload-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/delimited-payload-tokenfilter.asciidoc
index f50eb2fdd05..1cebf950338 100644
--- a/docs/reference/analysis/tokenfilters/delimited-payload-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/delimited-payload-tokenfilter.asciidoc
@@ -3,6 +3,13 @@
 
 Named `delimited_payload`. Splits tokens into tokens and payload whenever a delimiter character is found.
 
+[WARNING]
+============================================
+
+The older name `delimited_payload_filter` is deprecated and should not be used for new indices. Use `delimited_payload` instead.
+
+============================================
+
 Example: "the|1 quick|2 fox|3" is split by default into tokens `the`, `quick`, and `fox` with payloads `1`, `2`, and `3` respectively.
 
 Parameters:
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyDelimitedPayloadTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyDelimitedPayloadTokenFilterFactory.java
index d4ecee4b90b..484c9d9b128 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyDelimitedPayloadTokenFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyDelimitedPayloadTokenFilterFactory.java
@@ -32,7 +32,7 @@ public class LegacyDelimitedPayloadTokenFilterFactory extends DelimitedPayloadTo
 
     LegacyDelimitedPayloadTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, env, name, settings);
-        if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_7_0_0_alpha1)) {
+        if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_6_2_0)) {
             DEPRECATION_LOGGER.deprecated("Deprecated [delimited_payload_filter] used, replaced by [delimited_payload]");
         }
     }
diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml
index 3738e316b71..bfb6c97c24f 100644
--- a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml
+++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml
@@ -1028,8 +1028,8 @@
 ---
 "delimited_payload_filter":
     - skip:
-        version: " - 6.99.99"
-        reason:  delimited_payload_filter deprecated in 7.0, replaced by delimited_payload
+        version: " - 6.1.99"
+        reason:  delimited_payload_filter deprecated in 6.2, replaced by delimited_payload
         features: "warnings"
 
     - do:

From 7a445964460b7486691db8af2ad0447f3a375358 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20Lindstr=C3=B6m?= 
Date: Mon, 4 Dec 2017 10:29:08 +0100
Subject: [PATCH 155/297] Catch InvalidPathException in
 IcuCollationTokenFilterFactory (#27202)

Using custom rules in the icu_collation filter can fail on Windows. If the rules are interpreted
as a file location, this leads to an InvalidPathException when trying to read the rules from a file.
---
 .../IcuCollationTokenFilterFactory.java         |  3 ++-
 .../SimpleIcuCollationTokenFilterTests.java     | 17 +++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java
index 220e4448559..d48714ffaba 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.analysis;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.elasticsearch.common.io.Streams;
@@ -55,7 +56,7 @@ public class IcuCollationTokenFilterFactory extends AbstractTokenFilterFactory {
             Exception failureToResolve = null;
             try {
                 rules = Streams.copyToString(Files.newBufferedReader(environment.configFile().resolve(rules), Charset.forName("UTF-8")));
-            } catch (IOException | SecurityException e) {
+            } catch (IOException | SecurityException | InvalidPathException e) {
                 failureToResolve = e;
             }
             try {
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
index f0689bd1db9..3658e706cbf 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
@@ -218,6 +218,23 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
         TokenFilterFactory filterFactory = analysis.tokenFilter.get("myCollator");
         assertCollatesToSame(filterFactory, "Töne", "Toene");
     }
+    
+    /*
+     * Test a basic custom rules (should not interfere with reading rules list
+     * in IcuCollationTokenFilterFactory and throw InvalidPathException on
+     * Windows platforms).
+     */
+    public void testBasicCustomRules() throws Exception {
+        Settings settings = Settings.builder()
+                .put("index.analysis.filter.myCollator.type", "icu_collation")
+                .put("index.analysis.filter.myCollator.rules", "&a < g")
+                .build();
+        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
+
+        TokenFilterFactory filterFactory = analysis.tokenFilter.get("myCollator");
+        assertCollation(filterFactory, "green", "bird", -1);
+    }
+
 
     private void assertCollatesToSame(TokenFilterFactory factory, String string1, String string2) throws IOException {
         assertCollation(factory, string1, string2, 0);

From 72d0de4197d6370873f14f8a15396dc4bd30652d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?=
 <10398885+cbuescher@users.noreply.github.com>
Date: Mon, 4 Dec 2017 10:54:03 +0100
Subject: [PATCH 156/297] Add search window parameter k to MRR and DCG metric
 (#27595)

---
 .../rankeval/DiscountedCumulativeGain.java    |  47 +++++--
 .../index/rankeval/MeanReciprocalRank.java    |  50 +++++---
 .../index/rankeval/PrecisionAtK.java          |   6 +-
 .../DiscountedCumulativeGainTests.java        |  62 ++++++---
 .../rankeval/MeanReciprocalRankTests.java     |  34 ++++-
 .../index/rankeval/RankEvalRequestIT.java     | 118 +++++++++++++++---
 6 files changed, 248 insertions(+), 69 deletions(-)

diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java
index 141d45c274b..64d4ada0dc1 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java
@@ -33,21 +33,29 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings;
 
 /**
- * Metric implementing Discounted Cumulative Gain (https://en.wikipedia.org/wiki/Discounted_cumulative_gain).
    
+ * Metric implementing Discounted Cumulative Gain.
  * The `normalize` parameter can be set to calculate the normalized NDCG (set to false by default).
    
  * The optional `unknown_doc_rating` parameter can be used to specify a default rating for unlabeled documents.
+ * @see Discounted Cumulative Gain
    
  */
 public class DiscountedCumulativeGain implements EvaluationMetric {
 
     /** If set to true, the dcg will be normalized (ndcg) */
     private final boolean normalize;
 
+    /** the default search window size */
+    private static final int DEFAULT_K = 10;
+
+    /** the search window size */
+    private final int k;
+
     /**
      * Optional. If set, this will be the rating for docs that are unrated in the ranking evaluation request
      */
@@ -57,7 +65,7 @@ public class DiscountedCumulativeGain implements EvaluationMetric {
     private static final double LOG2 = Math.log(2.0);
 
     public DiscountedCumulativeGain() {
-        this(false, null);
+        this(false, null, DEFAULT_K);
     }
 
     /**
@@ -65,23 +73,27 @@ public class DiscountedCumulativeGain implements EvaluationMetric {
      *            If set to true, dcg will be normalized (ndcg) See
      *            https://en.wikipedia.org/wiki/Discounted_cumulative_gain
      * @param unknownDocRating
-     *            the rating for docs the user hasn't supplied an explicit
+     *            the rating for documents the user hasn't supplied an explicit
      *            rating for
+     * @param k the search window size all request use.
      */
-    public DiscountedCumulativeGain(boolean normalize, Integer unknownDocRating) {
+    public DiscountedCumulativeGain(boolean normalize, Integer unknownDocRating, int k) {
         this.normalize = normalize;
         this.unknownDocRating = unknownDocRating;
+        this.k = k;
     }
 
     DiscountedCumulativeGain(StreamInput in) throws IOException {
         normalize = in.readBoolean();
         unknownDocRating = in.readOptionalVInt();
+        k = in.readVInt();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(normalize);
         out.writeOptionalVInt(unknownDocRating);
+        out.writeVInt(k);
     }
 
     @Override
@@ -89,13 +101,14 @@ public class DiscountedCumulativeGain implements EvaluationMetric {
         return NAME;
     }
 
-    /**
-     * check whether this metric computes only dcg or "normalized" ndcg
-     */
-    public boolean getNormalize() {
+    boolean getNormalize() {
         return this.normalize;
     }
 
+    int getK() {
+        return this.k;
+    }
+
     /**
      * get the rating used for unrated documents
      */
@@ -103,6 +116,12 @@ public class DiscountedCumulativeGain implements EvaluationMetric {
         return this.unknownDocRating;
     }
 
+
+    @Override
+    public Optional forcedSearchSize() {
+        return Optional.of(k);
+    }
+
     @Override
     public EvalQueryQuality evaluate(String taskId, SearchHit[] hits,
             List ratedDocs) {
@@ -142,17 +161,21 @@ public class DiscountedCumulativeGain implements EvaluationMetric {
         return dcg;
     }
 
+    private static final ParseField K_FIELD = new ParseField("k");
     private static final ParseField NORMALIZE_FIELD = new ParseField("normalize");
     private static final ParseField UNKNOWN_DOC_RATING_FIELD = new ParseField("unknown_doc_rating");
     private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("dcg_at",
             args -> {
                 Boolean normalized = (Boolean) args[0];
-                return new DiscountedCumulativeGain(normalized == null ? false : normalized, (Integer) args[1]);
+                Integer optK = (Integer) args[2];
+                return new DiscountedCumulativeGain(normalized == null ? false : normalized, (Integer) args[1],
+                        optK == null ? DEFAULT_K : optK);
             });
 
     static {
         PARSER.declareBoolean(optionalConstructorArg(), NORMALIZE_FIELD);
         PARSER.declareInt(optionalConstructorArg(), UNKNOWN_DOC_RATING_FIELD);
+        PARSER.declareInt(optionalConstructorArg(), K_FIELD);
     }
 
     public static DiscountedCumulativeGain fromXContent(XContentParser parser) {
@@ -167,6 +190,7 @@ public class DiscountedCumulativeGain implements EvaluationMetric {
         if (unknownDocRating != null) {
             builder.field(UNKNOWN_DOC_RATING_FIELD.getPreferredName(), this.unknownDocRating);
         }
+        builder.field(K_FIELD.getPreferredName(), this.k);
         builder.endObject();
         builder.endObject();
         return builder;
@@ -182,11 +206,12 @@ public class DiscountedCumulativeGain implements EvaluationMetric {
         }
         DiscountedCumulativeGain other = (DiscountedCumulativeGain) obj;
         return Objects.equals(normalize, other.normalize)
-                && Objects.equals(unknownDocRating, other.unknownDocRating);
+                && Objects.equals(unknownDocRating, other.unknownDocRating)
+                && Objects.equals(k, other.k);
     }
 
     @Override
     public final int hashCode() {
-        return Objects.hash(normalize, unknownDocRating);
+        return Objects.hash(normalize, unknownDocRating, k);
     }
 }
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java
index 057bff6e147..a74fd8da3e6 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java
@@ -42,37 +42,57 @@ import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRati
  */
 public class MeanReciprocalRank implements EvaluationMetric {
 
-    private static final int DEFAULT_RATING_THRESHOLD = 1;
-
     public static final String NAME = "mean_reciprocal_rank";
 
-    /** ratings equal or above this value will be considered relevant. */
+    private static final int DEFAULT_RATING_THRESHOLD = 1;
+    private static final int DEFAULT_K = 10;
+
+    /** the search window size */
+    private final int k;
+
+    /** ratings equal or above this value will be considered relevant */
     private final int relevantRatingThreshhold;
 
     public MeanReciprocalRank() {
-        this(DEFAULT_RATING_THRESHOLD);
+        this(DEFAULT_RATING_THRESHOLD, DEFAULT_K);
     }
 
     MeanReciprocalRank(StreamInput in) throws IOException {
         this.relevantRatingThreshhold = in.readVInt();
+        this.k = in.readVInt();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(relevantRatingThreshhold);
+        out.writeVInt(this.relevantRatingThreshhold);
+        out.writeVInt(this.k);
     }
 
     /**
      * Metric implementing Mean Reciprocal Rank (https://en.wikipedia.org/wiki/Mean_reciprocal_rank).
    
-     * @param relevantRatingThreshold the rating value that a document needs to be regarded as "relevalnt". Defaults to 1.
+     * @param relevantRatingThreshold the rating value that a document needs to be regarded as "relevant". Defaults to 1.
+     * @param k the search window size all request use.
      */
-    public MeanReciprocalRank(int relevantRatingThreshold) {
+    public MeanReciprocalRank(int relevantRatingThreshold, int k) {
         if (relevantRatingThreshold < 0) {
             throw new IllegalArgumentException("Relevant rating threshold for precision must be positive integer.");
         }
+        if (k <= 0) {
+            throw new IllegalArgumentException("Window size k must be positive.");
+        }
+        this.k = k;
         this.relevantRatingThreshhold = relevantRatingThreshold;
     }
 
+    int getK() {
+        return this.k;
+    }
+
+    @Override
+    public Optional forcedSearchSize() {
+        return Optional.of(k);
+    }
+
     @Override
     public String getWriteableName() {
         return NAME;
@@ -113,18 +133,18 @@ public class MeanReciprocalRank implements EvaluationMetric {
     }
 
     private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold");
+    private static final ParseField K_FIELD = new ParseField("k");
     private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("reciprocal_rank",
             args -> {
                 Integer optionalThreshold = (Integer) args[0];
-                if (optionalThreshold == null) {
-                    return new MeanReciprocalRank();
-                } else {
-                    return new MeanReciprocalRank(optionalThreshold);
-                }
+                Integer optionalK = (Integer) args[1];
+                return new MeanReciprocalRank(optionalThreshold == null ? DEFAULT_RATING_THRESHOLD : optionalThreshold,
+                        optionalK == null ? DEFAULT_K : optionalK);
             });
 
     static {
         PARSER.declareInt(optionalConstructorArg(), RELEVANT_RATING_FIELD);
+        PARSER.declareInt(optionalConstructorArg(), K_FIELD);
     }
 
     public static MeanReciprocalRank fromXContent(XContentParser parser) {
@@ -136,6 +156,7 @@ public class MeanReciprocalRank implements EvaluationMetric {
         builder.startObject();
         builder.startObject(NAME);
         builder.field(RELEVANT_RATING_FIELD.getPreferredName(), this.relevantRatingThreshhold);
+        builder.field(K_FIELD.getPreferredName(), this.k);
         builder.endObject();
         builder.endObject();
         return builder;
@@ -150,12 +171,13 @@ public class MeanReciprocalRank implements EvaluationMetric {
             return false;
         }
         MeanReciprocalRank other = (MeanReciprocalRank) obj;
-        return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold);
+        return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold)
+                && Objects.equals(k, other.k);
     }
 
     @Override
     public final int hashCode() {
-        return Objects.hash(relevantRatingThreshhold);
+        return Objects.hash(relevantRatingThreshhold, k);
     }
 
     static class Breakdown implements MetricDetails {
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java
index 4beeeea2b40..63bdcb7307d 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java
@@ -96,7 +96,7 @@ public class PrecisionAtK implements EvaluationMetric {
                 Integer k = (Integer) args[2];
                 return new PrecisionAtK(threshHold == null ? 1 : threshHold,
                         ignoreUnlabeled == null ? false : ignoreUnlabeled,
-                                k == null ? 10 : k);
+                                k == null ? DEFAULT_K : k);
             });
 
     static {
@@ -111,6 +111,10 @@ public class PrecisionAtK implements EvaluationMetric {
         k = in.readVInt();
     }
 
+    int getK() {
+        return this.k;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(relevantRatingThreshhold);
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
index f3c38b7ae64..00f8a3018d4 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
@@ -42,13 +42,18 @@ import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashC
 
 public class DiscountedCumulativeGainTests extends ESTestCase {
 
+    static final double EXPECTED_DCG = 13.84826362927298;
+    static final double EXPECTED_IDCG = 14.595390756454922;
+    static final double EXPECTED_NDCG = EXPECTED_DCG / EXPECTED_IDCG;
+    private static final double DELTA = 10E-16;
+
     /**
      * Assuming the docs are ranked in the following order:
      *
      * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1)
      * -------------------------------------------------------------------------------------------
-     * 1 | 3 | 7.0 | 1.0 | 7.0 2 | 
-     * 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721
+     * 1 | 3 | 7.0 | 1.0 | 7.0 | 7.0 | 
+     * 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721
      * 3 | 3 | 7.0 | 2.0 | 3.5
      * 4 | 0 | 0.0 | 2.321928094887362 | 0.0
      * 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163
@@ -66,7 +71,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
             hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
         }
         DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
-        assertEquals(13.84826362927298, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001);
+        assertEquals(EXPECTED_DCG, dcg.evaluate("id", hits, rated).getQualityLevel(), DELTA);
 
         /**
          * Check with normalization: to get the maximal possible dcg, sort documents by
@@ -83,8 +88,8 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
          *
          * idcg = 14.595390756454922 (sum of last column)
          */
-        dcg = new DiscountedCumulativeGain(true, null);
-        assertEquals(13.84826362927298 / 14.595390756454922, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001);
+        dcg = new DiscountedCumulativeGain(true, null, 10);
+        assertEquals(EXPECTED_NDCG, dcg.evaluate("id", hits, rated).getQualityLevel(), DELTA);
     }
 
     /**
@@ -117,7 +122,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
         }
         DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
         EvalQueryQuality result = dcg.evaluate("id", hits, rated);
-        assertEquals(12.779642067948913, result.getQualityLevel(), 0.00001);
+        assertEquals(12.779642067948913, result.getQualityLevel(), DELTA);
         assertEquals(2, filterUnknownDocuments(result.getHitsAndRatings()).size());
 
         /**
@@ -135,8 +140,8 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
          *
          * idcg = 13.347184833073591 (sum of last column)
          */
-        dcg = new DiscountedCumulativeGain(true, null);
-        assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001);
+        dcg = new DiscountedCumulativeGain(true, null, 10);
+        assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), DELTA);
     }
 
     /**
@@ -174,7 +179,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
         }
         DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
         EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs);
-        assertEquals(12.392789260714371, result.getQualityLevel(), 0.00001);
+        assertEquals(12.392789260714371, result.getQualityLevel(), DELTA);
         assertEquals(1, filterUnknownDocuments(result.getHitsAndRatings()).size());
 
         /**
@@ -193,16 +198,27 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
          *
          * idcg = 13.347184833073591 (sum of last column)
          */
-        dcg = new DiscountedCumulativeGain(true, null);
-        assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), 0.00001);
+        dcg = new DiscountedCumulativeGain(true, null, 10);
+        assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), DELTA);
     }
 
     public void testParseFromXContent() throws IOException {
-        String xContent = " { \"unknown_doc_rating\": 2, \"normalize\": true }";
+        assertParsedCorrect("{ \"unknown_doc_rating\": 2, \"normalize\": true, \"k\" : 15 }", 2, true, 15);
+        assertParsedCorrect("{ \"normalize\": false, \"k\" : 15 }", null, false, 15);
+        assertParsedCorrect("{ \"unknown_doc_rating\": 2, \"k\" : 15 }", 2, false, 15);
+        assertParsedCorrect("{ \"unknown_doc_rating\": 2, \"normalize\": true }", 2, true, 10);
+        assertParsedCorrect("{ \"normalize\": true }", null, true, 10);
+        assertParsedCorrect("{ \"k\": 23 }", null, false, 23);
+        assertParsedCorrect("{ \"unknown_doc_rating\": 2 }", 2, false, 10);
+    }
+
+    private void assertParsedCorrect(String xContent, Integer expectedUnknownDocRating, boolean expectedNormalize, int expectedK)
+            throws IOException {
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
             DiscountedCumulativeGain dcgAt = DiscountedCumulativeGain.fromXContent(parser);
-            assertEquals(2, dcgAt.getUnknownDocRating().intValue());
-            assertEquals(true, dcgAt.getNormalize());
+            assertEquals(expectedUnknownDocRating, dcgAt.getUnknownDocRating());
+            assertEquals(expectedNormalize, dcgAt.getNormalize());
+            assertEquals(expectedK, dcgAt.getK());
         }
     }
 
@@ -210,7 +226,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
         boolean normalize = randomBoolean();
         Integer unknownDocRating = new Integer(randomIntBetween(0, 1000));
 
-        return new DiscountedCumulativeGain(normalize, unknownDocRating);
+        return new DiscountedCumulativeGain(normalize, unknownDocRating, 10);
     }
 
     public void testXContentRoundtrip() throws IOException {
@@ -238,16 +254,22 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
 
     public void testEqualsAndHash() throws IOException {
         checkEqualsAndHashCode(createTestItem(), original -> {
-            return new DiscountedCumulativeGain(original.getNormalize(), original.getUnknownDocRating());
+            return new DiscountedCumulativeGain(original.getNormalize(), original.getUnknownDocRating(), original.getK());
         }, DiscountedCumulativeGainTests::mutateTestItem);
     }
 
     private static DiscountedCumulativeGain mutateTestItem(DiscountedCumulativeGain original) {
-        if (randomBoolean()) {
-            return new DiscountedCumulativeGain(!original.getNormalize(), original.getUnknownDocRating());
-        } else {
+        switch (randomIntBetween(0, 2)) {
+        case 0:
+            return new DiscountedCumulativeGain(!original.getNormalize(), original.getUnknownDocRating(), original.getK());
+        case 1:
             return new DiscountedCumulativeGain(original.getNormalize(),
-                    randomValueOtherThan(original.getUnknownDocRating(), () -> randomIntBetween(0, 10)));
+                    randomValueOtherThan(original.getUnknownDocRating(), () -> randomIntBetween(0, 10)), original.getK());
+        case 2:
+            return new DiscountedCumulativeGain(original.getNormalize(), original.getUnknownDocRating(),
+                    randomValueOtherThan(original.getK(), () -> randomIntBetween(1, 10)));
+        default:
+            throw new IllegalArgumentException("mutation variant not allowed");
         }
     }
 }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java
index 3a1b4939758..42f7e32671f 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java
@@ -48,12 +48,28 @@ public class MeanReciprocalRankTests extends ESTestCase {
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
             MeanReciprocalRank mrr = MeanReciprocalRank.fromXContent(parser);
             assertEquals(1, mrr.getRelevantRatingThreshold());
+            assertEquals(10, mrr.getK());
         }
 
         xContent = "{ \"relevant_rating_threshold\": 2 }";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
             MeanReciprocalRank mrr = MeanReciprocalRank.fromXContent(parser);
             assertEquals(2, mrr.getRelevantRatingThreshold());
+            assertEquals(10, mrr.getK());
+        }
+
+        xContent = "{ \"relevant_rating_threshold\": 2, \"k\" : 15 }";
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
+            MeanReciprocalRank mrr = MeanReciprocalRank.fromXContent(parser);
+            assertEquals(2, mrr.getRelevantRatingThreshold());
+            assertEquals(15, mrr.getK());
+        }
+
+        xContent = "{ \"k\" : 15 }";
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
+            MeanReciprocalRank mrr = MeanReciprocalRank.fromXContent(parser);
+            assertEquals(1, mrr.getRelevantRatingThreshold());
+            assertEquals(15, mrr.getK());
         }
     }
 
@@ -116,7 +132,7 @@ public class MeanReciprocalRankTests extends ESTestCase {
         rated.add(new RatedDocument("test",  "4", 4));
         SearchHit[] hits = createSearchHits(0, 5, "test");
 
-        MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(2);
+        MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(2, 10);
         EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, rated);
         assertEquals((double) 1 / 3, evaluation.getQualityLevel(), 0.00001);
         assertEquals(3, ((MeanReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
@@ -167,7 +183,7 @@ public class MeanReciprocalRankTests extends ESTestCase {
     }
 
     static MeanReciprocalRank createTestItem() {
-        return new MeanReciprocalRank(randomIntBetween(0, 20));
+        return new MeanReciprocalRank(randomIntBetween(0, 20), randomIntBetween(1, 20));
     }
 
     public void testSerialization() throws IOException {
@@ -184,14 +200,22 @@ public class MeanReciprocalRankTests extends ESTestCase {
     }
 
     private static MeanReciprocalRank copy(MeanReciprocalRank testItem) {
-        return new MeanReciprocalRank(testItem.getRelevantRatingThreshold());
+        return new MeanReciprocalRank(testItem.getRelevantRatingThreshold(), testItem.getK());
     }
 
     private static MeanReciprocalRank mutate(MeanReciprocalRank testItem) {
-        return new MeanReciprocalRank(randomValueOtherThan(testItem.getRelevantRatingThreshold(), () -> randomIntBetween(0, 10)));
+        if (randomBoolean()) {
+            return new MeanReciprocalRank(testItem.getRelevantRatingThreshold() + 1, testItem.getK());
+        } else {
+            return new MeanReciprocalRank(testItem.getRelevantRatingThreshold(), testItem.getK() + 1);
+        }
     }
 
     public void testInvalidRelevantThreshold() {
-        expectThrows(IllegalArgumentException.class, () -> new MeanReciprocalRank(-1));
+        expectThrows(IllegalArgumentException.class, () -> new MeanReciprocalRank(-1, 1));
+    }
+
+    public void testInvalidK() {
+        expectThrows(IllegalArgumentException.class, () -> new MeanReciprocalRank(1, -1));
     }
 }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
index 2ab3e2d9a57..1c10da61fa1 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
@@ -20,18 +20,17 @@
 package org.elasticsearch.index.rankeval;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.Before;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -64,13 +63,16 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         refresh();
     }
 
+    /**
+     * Test cases retrieves all six documents indexed above. The first part checks the Prec@10 calculation where
+     * all unlabeled docs are treated as "unrelevant". We average Prec@ metric across two search use cases, the
+     * first one that labels 4 out of the 6 documents as relevant, the second one with only one relevant document.
+     */
     public void testPrecisionAtRequest() {
-        List indices = Arrays.asList(new String[] { "test" });
-
         List specifications = new ArrayList<>();
         SearchSourceBuilder testQuery = new SearchSourceBuilder();
         testQuery.query(new MatchAllQueryBuilder());
-        testQuery.sort(FieldSortBuilder.DOC_FIELD_NAME);
+        testQuery.sort("_id");
         RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query",
                 createRelevant("2", "3", "4", "5"), testQuery);
         amsterdamRequest.addSummaryFields(Arrays.asList(new String[] { "text", "title" }));
@@ -79,12 +81,11 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("1"),
                 testQuery);
         berlinRequest.addSummaryFields(Arrays.asList(new String[] { "text", "title" }));
-
         specifications.add(berlinRequest);
 
         PrecisionAtK metric = new PrecisionAtK(1, false, 10);
         RankEvalSpec task = new RankEvalSpec(specifications, metric);
-        task.addIndices(indices);
+        task.addIndices(Collections.singletonList("test"));
 
         RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(),
                 RankEvalAction.INSTANCE, new RankEvalRequest());
@@ -92,6 +93,8 @@ public class RankEvalRequestIT extends ESIntegTestCase {
 
         RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request())
                 .actionGet();
+        // the expected Prec@ for the first query is 4/6 and the expected Prec@ for the
+        // second is 1/6, divided by 2 to get the average
         double expectedPrecision = (1.0 / 6.0 + 4.0 / 6.0) / 2.0;
         assertEquals(expectedPrecision, response.getEvaluationResult(), Double.MIN_VALUE);
         Set> entrySet = response.getPartialResults().entrySet();
@@ -129,14 +132,96 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         // test that a different window size k affects the result
         metric = new PrecisionAtK(1, false, 3);
         task = new RankEvalSpec(specifications, metric);
-        task.addIndices(indices);
+        task.addIndices(Collections.singletonList("test"));
 
         builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
         builder.setRankEvalSpec(task);
 
         response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
+        // if we look only at top 3 documente, the expected P@3 for the first query is
+        // 2/3 and the expected Prec@ for the second is 1/3, divided by 2 to get the average
         expectedPrecision = (1.0 / 3.0 + 2.0 / 3.0) / 2.0;
-        assertEquals(0.5, response.getEvaluationResult(), Double.MIN_VALUE);
+        assertEquals(expectedPrecision, response.getEvaluationResult(), Double.MIN_VALUE);
+    }
+
+    /**
+     * This test assumes we are using the same ratings as in {@link DiscountedCumulativeGainTests#testDCGAt()}.
+     * See details in that test case for how the expected values are calculated
+     */
+    public void testDCGRequest() {
+        SearchSourceBuilder testQuery = new SearchSourceBuilder();
+        testQuery.query(new MatchAllQueryBuilder());
+        testQuery.sort("_id");
+
+        List specifications = new ArrayList<>();
+        List ratedDocs = Arrays.asList(
+                new RatedDocument("test", "1", 3),
+                new RatedDocument("test", "2", 2),
+                new RatedDocument("test", "3", 3),
+                new RatedDocument("test", "4", 0),
+                new RatedDocument("test", "5", 1),
+                new RatedDocument("test", "6", 2));
+        specifications.add(new RatedRequest("amsterdam_query", ratedDocs, testQuery));
+
+        DiscountedCumulativeGain metric = new DiscountedCumulativeGain(false, null, 10);
+        RankEvalSpec task = new RankEvalSpec(specifications, metric);
+        task.addIndices(Collections.singletonList("test"));
+
+        RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
+        builder.setRankEvalSpec(task);
+
+        RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
+        assertEquals(DiscountedCumulativeGainTests.EXPECTED_DCG, response.getEvaluationResult(), Double.MIN_VALUE);
+
+        // test that a different window size k affects the result
+        metric = new DiscountedCumulativeGain(false, null, 3);
+        task = new RankEvalSpec(specifications, metric);
+        task.addIndices(Collections.singletonList("test"));
+
+        builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
+        builder.setRankEvalSpec(task);
+
+        response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
+        assertEquals(12.392789260714371, response.getEvaluationResult(), Double.MIN_VALUE);
+    }
+
+    public void testMRRRequest() {
+        SearchSourceBuilder testQuery = new SearchSourceBuilder();
+        testQuery.query(new MatchAllQueryBuilder());
+        testQuery.sort("_id");
+
+        List specifications = new ArrayList<>();
+        specifications.add(new RatedRequest("amsterdam_query", createRelevant("5"), testQuery));
+        specifications.add(new RatedRequest("berlin_query", createRelevant("1"), testQuery));
+
+        MeanReciprocalRank metric = new MeanReciprocalRank(1, 10);
+        RankEvalSpec task = new RankEvalSpec(specifications, metric);
+        task.addIndices(Collections.singletonList("test"));
+
+        RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
+        builder.setRankEvalSpec(task);
+
+        RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
+        // the expected reciprocal rank for the amsterdam_query is 1/5
+        // the expected reciprocal rank for the berlin_query is 1/1
+        // dividing by 2 to get the average
+        double expectedMRR = (1.0 / 1.0 + 1.0 / 5.0) / 2.0;
+        assertEquals(expectedMRR, response.getEvaluationResult(), 0.0);
+
+        // test that a different window size k affects the result
+        metric = new MeanReciprocalRank(1, 3);
+        task = new RankEvalSpec(specifications, metric);
+        task.addIndices(Collections.singletonList("test"));
+
+        builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
+        builder.setRankEvalSpec(task);
+
+        response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
+        // limiting to top 3 results, the amsterdam_query has no relevant document in it
+        // the reciprocal rank for the berlin_query is 1/1
+        // dividing by 2 to get the average
+        expectedMRR = (1.0/ 1.0) / 2.0;
+        assertEquals(expectedMRR, response.getEvaluationResult(), 0.0);
     }
 
     /**
@@ -162,16 +247,13 @@ public class RankEvalRequestIT extends ESIntegTestCase {
         RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK());
         task.addIndices(indices);
 
-        try (Client client = client()) {
-            RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client, RankEvalAction.INSTANCE, new RankEvalRequest());
-            builder.setRankEvalSpec(task);
+        RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
+        builder.setRankEvalSpec(task);
 
-            RankEvalResponse response = client.execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
-            assertEquals(1, response.getFailures().size());
-            ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(response.getFailures().get("broken_query"));
-            assertEquals("java.lang.NumberFormatException: For input string: \"noStringOnNumericFields\"",
-                    rootCauses[0].getCause().toString());
-        }
+        RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
+        assertEquals(1, response.getFailures().size());
+        ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(response.getFailures().get("broken_query"));
+        assertEquals("java.lang.NumberFormatException: For input string: \"noStringOnNumericFields\"", rootCauses[0].getCause().toString());
     }
 
     private static List createRelevant(String... docs) {

From b44ae25c27be0a02b015b54db2862540e93f3765 Mon Sep 17 00:00:00 2001
From: Catalin Ursachi 
Date: Mon, 4 Dec 2017 10:51:00 +0000
Subject: [PATCH 157/297] Updated "string" datatype in docs & tests to "text"
 (#27629)

---
 .../cluster/metadata/MetaDataMappingServiceTests.java     | 4 ++--
 docs/java-api/admin/indices/put-mapping.asciidoc          | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java
index 7385387305f..428d9488dc2 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java
@@ -98,7 +98,7 @@ public class MetaDataMappingServiceTests extends ESSingleNodeTestCase {
         final ClusterService clusterService = getInstanceFromNode(ClusterService.class);
         // TODO - it will be nice to get a random mapping generator
         final PutMappingClusterStateUpdateRequest request = new PutMappingClusterStateUpdateRequest().type("type");
-        request.source("{ \"properties\" { \"field\": { \"type\": \"string\" }}}");
+        request.source("{ \"properties\" { \"field\": { \"type\": \"text\" }}}");
         mappingService.putMappingExecutor.execute(clusterService.state(), Collections.singletonList(request));
         assertThat(indexService.mapperService().documentMapper("type").mappingSource(), equalTo(currentMapping));
     }
@@ -109,7 +109,7 @@ public class MetaDataMappingServiceTests extends ESSingleNodeTestCase {
         final MetaDataMappingService mappingService = getInstanceFromNode(MetaDataMappingService.class);
         final ClusterService clusterService = getInstanceFromNode(ClusterService.class);
         final PutMappingClusterStateUpdateRequest request = new PutMappingClusterStateUpdateRequest().type("type");
-        request.source("{ \"properties\" { \"field\": { \"type\": \"string\" }}}");
+        request.source("{ \"properties\" { \"field\": { \"type\": \"text\" }}}");
         ClusterState result = mappingService.putMappingExecutor.execute(clusterService.state(), Collections.singletonList(request))
             .resultingState;
 
diff --git a/docs/java-api/admin/indices/put-mapping.asciidoc b/docs/java-api/admin/indices/put-mapping.asciidoc
index 6c2a5406528..e52c66d96c3 100644
--- a/docs/java-api/admin/indices/put-mapping.asciidoc
+++ b/docs/java-api/admin/indices/put-mapping.asciidoc
@@ -10,7 +10,7 @@ client.admin().indices().prepareCreate("twitter")   <1>
                 "    \"tweet\": {\n" +
                 "      \"properties\": {\n" +
                 "        \"message\": {\n" +
-                "          \"type\": \"string\"\n" +
+                "          \"type\": \"text\"\n" +
                 "        }\n" +
                 "      }\n" +
                 "    }\n" +
@@ -30,7 +30,7 @@ client.admin().indices().preparePutMapping("twitter")   <1>
         .setSource("{\n" +                              <3>
                 "  \"properties\": {\n" +
                 "    \"name\": {\n" +
-                "      \"type\": \"string\"\n" +
+                "      \"type\": \"text\"\n" +
                 "    }\n" +
                 "  }\n" +
                 "}")
@@ -43,7 +43,7 @@ client.admin().indices().preparePutMapping("twitter")
                 "    \"user\":{\n" +                        <4>
                 "        \"properties\": {\n" +
                 "            \"name\": {\n" +
-                "                \"type\": \"string\"\n" +
+                "                \"type\": \"text\"\n" +
                 "            }\n" +
                 "        }\n" +
                 "    }\n" +
@@ -64,7 +64,7 @@ client.admin().indices().preparePutMapping("twitter")   <1>
         .setSource("{\n" +                              <3>
                 "  \"properties\": {\n" +
                 "    \"user_name\": {\n" +
-                "      \"type\": \"string\"\n" +
+                "      \"type\": \"text\"\n" +
                 "    }\n" +
                 "  }\n" +
                 "}")

From 1a976ea7a4a3277cc853f828954eaeb411294a27 Mon Sep 17 00:00:00 2001
From: Boaz Leskes 
Date: Thu, 30 Nov 2017 15:07:06 +0100
Subject: [PATCH 158/297] Cherry pick tests and seqNo recovery hardning from
 #27580

---
 .../index/engine/InternalEngine.java          |   5 -
 .../elasticsearch/index/shard/IndexShard.java |  41 +++--
 .../recovery/RecoverySourceHandler.java       |  97 ++++++----
 .../RecoveryDuringReplicationTests.java       |  10 +-
 .../recovery/RecoverySourceHandlerTests.java  |  83 ++++++---
 .../upgrades/FullClusterRestartIT.java        |  26 +++
 .../elasticsearch/backwards/IndexingIT.java   |  27 +--
 .../elasticsearch/upgrades/RecoveryIT.java    | 168 +++++++++++++++++-
 .../test/old_cluster/10_basic.yml             |  10 ++
 .../test/upgraded_cluster/10_basic.yml        |   1 +
 .../test/rest/ESRestTestCase.java             |  17 +-
 11 files changed, 381 insertions(+), 104 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index e431bfb7a5b..fe8a33f5ecd 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -426,11 +426,6 @@ public class InternalEngine extends Engine {
         } else if (translog.isCurrent(translogGeneration) == false) {
             commitIndexWriter(indexWriter, translog, lastCommittedSegmentInfos.getUserData().get(Engine.SYNC_COMMIT_ID));
             refreshLastCommittedSegmentInfos();
-        } else if (lastCommittedSegmentInfos.getUserData().containsKey(HISTORY_UUID_KEY) == false)  {
-            assert historyUUID != null;
-            // put the history uuid into the index
-            commitIndexWriter(indexWriter, translog, lastCommittedSegmentInfos.getUserData().get(Engine.SYNC_COMMIT_ID));
-            refreshLastCommittedSegmentInfos();
         }
         // clean up what's not needed
         translog.trimUnreferencedReaders();
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 1dc28915d09..12da974645c 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -48,7 +48,6 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
@@ -66,7 +65,6 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AsyncIOProcessor;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexModule;
@@ -416,12 +414,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                     logger.debug("failed to refresh due to move to cluster wide started", e);
                 }
 
-                if (newRouting.primary()) {
-                    final DiscoveryNode recoverySourceNode = recoveryState.getSourceNode();
-                    if (currentRouting.isRelocationTarget() == false || recoverySourceNode.getVersion().before(Version.V_6_0_0_alpha1)) {
-                        // there was no primary context hand-off in < 6.0.0, need to manually activate the shard
-                        getEngine().seqNoService().activatePrimaryMode(getEngine().seqNoService().getLocalCheckpoint());
-                    }
+                if (newRouting.primary() && currentRouting.isRelocationTarget() == false) {
+                    // there was no primary context hand-off in < 6.0.0, need to manually activate the shard
+                    getEngine().seqNoService().activatePrimaryMode(getEngine().seqNoService().getLocalCheckpoint());
                 }
 
                 changeState(IndexShardState.STARTED, "global state is [" + newRouting.state() + "]");
@@ -485,15 +480,18 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                                  * subsequently fails before the primary/replica re-sync completes successfully and we are now being
                                  * promoted, the local checkpoint tracker here could be left in a state where it would re-issue sequence
                                  * numbers. To ensure that this is not the case, we restore the state of the local checkpoint tracker by
-                                 * replaying the translog and marking any operations there are completed. Rolling the translog generation is
-                                 * not strictly needed here (as we will never have collisions between sequence numbers in a translog
-                                 * generation in a new primary as it takes the last known sequence number as a starting point), but it
-                                 * simplifies reasoning about the relationship between primary terms and translog generations.
+                                 * replaying the translog and marking any operations there are completed.
                                  */
-                                getEngine().rollTranslogGeneration();
-                                getEngine().restoreLocalCheckpointFromTranslog();
-                                getEngine().fillSeqNoGaps(newPrimaryTerm);
-                                getEngine().seqNoService().updateLocalCheckpointForShard(currentRouting.allocationId().getId(),
+                                final Engine engine = getEngine();
+                                engine.restoreLocalCheckpointFromTranslog();
+                                /* Rolling the translog generation is not strictly needed here (as we will never have collisions between
+                                 * sequence numbers in a translog generation in a new primary as it takes the last known sequence number
+                                 * as a starting point), but it simplifies reasoning about the relationship between primary terms and
+                                 * translog generations.
+                                 */
+                                engine.rollTranslogGeneration();
+                                engine.fillSeqNoGaps(newPrimaryTerm);
+                                engine.seqNoService().updateLocalCheckpointForShard(currentRouting.allocationId().getId(),
                                     getEngine().seqNoService().getLocalCheckpoint());
                                 primaryReplicaSyncer.accept(this, new ActionListener() {
                                     @Override
@@ -1337,6 +1335,17 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             active.set(true);
             newEngine.recoverFromTranslog();
         }
+        assertSequenceNumbersInCommit();
+    }
+
+    private boolean assertSequenceNumbersInCommit() throws IOException {
+        final Map userData = SegmentInfos.readLatestCommit(store.directory()).getUserData();
+        assert userData.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) : "commit point doesn't contains a local checkpoint";
+        assert userData.containsKey(SequenceNumbers.MAX_SEQ_NO) : "commit point doesn't contains a maximum sequence number";
+        assert userData.containsKey(Engine.HISTORY_UUID_KEY) : "commit point doesn't contains a history uuid";
+        assert userData.get(Engine.HISTORY_UUID_KEY).equals(getHistoryUUID()) : "commit point history uuid ["
+            + userData.get(Engine.HISTORY_UUID_KEY) + "] is different than engine [" + getHistoryUUID() + "]";
+        return true;
     }
 
     private boolean assertMaxUnsafeAutoIdInCommit() throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index 71ad21c14d7..a847088869b 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -149,12 +149,13 @@ public class RecoverySourceHandler {
             final Translog translog = shard.getTranslog();
 
             final long startingSeqNo;
+            final long requiredSeqNoRangeStart;
             final boolean isSequenceNumberBasedRecoveryPossible = request.startingSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO &&
                 isTargetSameHistory() && isTranslogReadyForSequenceNumberBasedRecovery();
-
             if (isSequenceNumberBasedRecoveryPossible) {
                 logger.trace("performing sequence numbers based recovery. starting at [{}]", request.startingSeqNo());
                 startingSeqNo = request.startingSeqNo();
+                requiredSeqNoRangeStart = startingSeqNo;
             } else {
                 final Engine.IndexCommitRef phase1Snapshot;
                 try {
@@ -162,10 +163,12 @@ public class RecoverySourceHandler {
                 } catch (final Exception e) {
                     throw new RecoveryEngineException(shard.shardId(), 1, "snapshot failed", e);
                 }
-                // we set this to unassigned to create a translog roughly according to the retention policy
-                // on the target
-                startingSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
-
+                // we set this to 0 to create a translog roughly according to the retention policy
+                // on the target. Note that it will still filter out legacy operations with no sequence numbers
+                startingSeqNo = 0;
+                // but we must have everything above the local checkpoint in the commit
+                requiredSeqNoRangeStart =
+                    Long.parseLong(phase1Snapshot.getIndexCommit().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) + 1;
                 try {
                     phase1(phase1Snapshot.getIndexCommit(), translog::totalOperations);
                 } catch (final Exception e) {
@@ -178,6 +181,9 @@ public class RecoverySourceHandler {
                     }
                 }
             }
+            assert startingSeqNo >= 0 : "startingSeqNo must be non negative. got: " + startingSeqNo;
+            assert requiredSeqNoRangeStart >= startingSeqNo : "requiredSeqNoRangeStart [" + requiredSeqNoRangeStart + "] is lower than ["
+                + startingSeqNo + "]";
 
             runUnderPrimaryPermit(() -> shard.initiateTracking(request.targetAllocationId()));
 
@@ -187,10 +193,19 @@ public class RecoverySourceHandler {
                 throw new RecoveryEngineException(shard.shardId(), 1, "prepare target for translog failed", e);
             }
 
+            final long endingSeqNo = shard.seqNoStats().getMaxSeqNo();
+            /*
+             * We need to wait for all operations up to the current max to complete, otherwise we can not guarantee that all
+             * operations in the required range will be available for replaying from the translog of the source.
+             */
+            cancellableThreads.execute(() -> shard.waitForOpsToComplete(endingSeqNo));
+
+            logger.trace("all operations up to [{}] completed, which will be used as an ending sequence number", endingSeqNo);
+
             logger.trace("snapshot translog for recovery; current size is [{}]", translog.estimateTotalOperationsFromMinSeq(startingSeqNo));
             final long targetLocalCheckpoint;
             try(Translog.Snapshot snapshot = translog.newSnapshotFromMinSeqNo(startingSeqNo)) {
-                targetLocalCheckpoint = phase2(startingSeqNo, snapshot);
+                targetLocalCheckpoint = phase2(startingSeqNo, requiredSeqNoRangeStart, endingSeqNo, snapshot);
             } catch (Exception e) {
                 throw new RecoveryEngineException(shard.shardId(), 2, "phase2 failed", e);
             }
@@ -224,7 +239,8 @@ public class RecoverySourceHandler {
 
     /**
      * Determines if the source translog is ready for a sequence-number-based peer recovery. The main condition here is that the source
-     * translog contains all operations between the local checkpoint on the target and the current maximum sequence number on the source.
+     * translog contains all operations above the local checkpoint on the target. We already know the that translog contains or will contain
+     * all ops above the source local checkpoint, so we can stop check there.
      *
      * @return {@code true} if the source is ready for a sequence-number-based recovery
      * @throws IOException if an I/O exception occurred reading the translog snapshot
@@ -232,18 +248,10 @@ public class RecoverySourceHandler {
     boolean isTranslogReadyForSequenceNumberBasedRecovery() throws IOException {
         final long startingSeqNo = request.startingSeqNo();
         assert startingSeqNo >= 0;
-        final long endingSeqNo = shard.seqNoStats().getMaxSeqNo();
-        logger.trace("testing sequence numbers in range: [{}, {}]", startingSeqNo, endingSeqNo);
+        final long localCheckpoint = shard.getLocalCheckpoint();
+        logger.trace("testing sequence numbers in range: [{}, {}]", startingSeqNo, localCheckpoint);
         // the start recovery request is initialized with the starting sequence number set to the target shard's local checkpoint plus one
-        if (startingSeqNo - 1 <= endingSeqNo) {
-            /*
-             * We need to wait for all operations up to the current max to complete, otherwise we can not guarantee that all
-             * operations in the required range will be available for replaying from the translog of the source.
-             */
-            cancellableThreads.execute(() -> shard.waitForOpsToComplete(endingSeqNo));
-
-            logger.trace("all operations up to [{}] completed, checking translog content", endingSeqNo);
-
+        if (startingSeqNo - 1 <= localCheckpoint) {
             final LocalCheckpointTracker tracker = new LocalCheckpointTracker(startingSeqNo, startingSeqNo - 1);
             try (Translog.Snapshot snapshot = shard.getTranslog().newSnapshotFromMinSeqNo(startingSeqNo)) {
                 Translog.Operation operation;
@@ -253,7 +261,7 @@ public class RecoverySourceHandler {
                     }
                 }
             }
-            return tracker.getCheckpoint() >= endingSeqNo;
+            return tracker.getCheckpoint() >= localCheckpoint;
         } else {
             return false;
         }
@@ -433,13 +441,15 @@ public class RecoverySourceHandler {
      * point-in-time view of the translog). It then sends each translog operation to the target node so it can be replayed into the new
      * shard.
      *
-     * @param startingSeqNo the sequence number to start recovery from, or {@link SequenceNumbers#UNASSIGNED_SEQ_NO} if all
-     *                      ops should be sent
-     * @param snapshot      a snapshot of the translog
-     *
+     * @param startingSeqNo           the sequence number to start recovery from, or {@link SequenceNumbers#UNASSIGNED_SEQ_NO} if all
+     *                                ops should be sent
+     * @param requiredSeqNoRangeStart the lower sequence number of the required range (ending with endingSeqNo)
+     * @param endingSeqNo             the highest sequence number that should be sent
+     * @param snapshot                a snapshot of the translog
      * @return the local checkpoint on the target
      */
-    long phase2(final long startingSeqNo, final Translog.Snapshot snapshot) throws IOException {
+    long phase2(final long startingSeqNo, long requiredSeqNoRangeStart, long endingSeqNo, final Translog.Snapshot snapshot)
+        throws IOException {
         if (shard.state() == IndexShardState.CLOSED) {
             throw new IndexShardClosedException(request.shardId());
         }
@@ -447,10 +457,11 @@ public class RecoverySourceHandler {
 
         final StopWatch stopWatch = new StopWatch().start();
 
-        logger.trace("recovery [phase2]: sending transaction log operations");
+        logger.trace("recovery [phase2]: sending transaction log operations (seq# from [" +  startingSeqNo  + "], " +
+            "required [" + requiredSeqNoRangeStart + ":" + endingSeqNo + "]");
 
         // send all the snapshot's translog operations to the target
-        final SendSnapshotResult result = sendSnapshot(startingSeqNo, snapshot);
+        final SendSnapshotResult result = sendSnapshot(startingSeqNo, requiredSeqNoRangeStart, endingSeqNo, snapshot);
 
         stopWatch.stop();
         logger.trace("recovery [phase2]: took [{}]", stopWatch.totalTime());
@@ -511,18 +522,26 @@ public class RecoverySourceHandler {
      * 
    
      * Operations are bulked into a single request depending on an operation count limit or size-in-bytes limit.
      *
-     * @param startingSeqNo the sequence number for which only operations with a sequence number greater than this will be sent
-     * @param snapshot      the translog snapshot to replay operations from
-     * @return the local checkpoint on the target and the total number of operations sent
+     * @param startingSeqNo           the sequence number for which only operations with a sequence number greater than this will be sent
+     * @param requiredSeqNoRangeStart the lower sequence number of the required range
+     * @param endingSeqNo             the upper bound of the sequence number range to be sent (inclusive)
+     * @param snapshot                the translog snapshot to replay operations from  @return the local checkpoint on the target and the
+     *                                total number of operations sent
      * @throws IOException if an I/O exception occurred reading the translog snapshot
      */
-    protected SendSnapshotResult sendSnapshot(final long startingSeqNo, final Translog.Snapshot snapshot) throws IOException {
+    protected SendSnapshotResult sendSnapshot(final long startingSeqNo, long requiredSeqNoRangeStart, long endingSeqNo,
+                                              final Translog.Snapshot snapshot) throws IOException {
+        assert requiredSeqNoRangeStart <= endingSeqNo + 1:
+            "requiredSeqNoRangeStart " + requiredSeqNoRangeStart + " is larger than endingSeqNo " + endingSeqNo;
+        assert startingSeqNo <= requiredSeqNoRangeStart :
+            "startingSeqNo " + startingSeqNo + " is larger than requiredSeqNoRangeStart " + requiredSeqNoRangeStart;
         int ops = 0;
         long size = 0;
         int skippedOps = 0;
         int totalSentOps = 0;
         final AtomicLong targetLocalCheckpoint = new AtomicLong(SequenceNumbers.UNASSIGNED_SEQ_NO);
         final List operations = new ArrayList<>();
+        final LocalCheckpointTracker requiredOpsTracker = new LocalCheckpointTracker(endingSeqNo, requiredSeqNoRangeStart - 1);
 
         final int expectedTotalOps = snapshot.totalOperations();
         if (expectedTotalOps == 0) {
@@ -539,12 +558,9 @@ public class RecoverySourceHandler {
                 throw new IndexShardClosedException(request.shardId());
             }
             cancellableThreads.checkForCancel();
-            /*
-             * If we are doing a sequence-number-based recovery, we have to skip older ops for which no sequence number was assigned, and
-             * any ops before the starting sequence number.
-             */
+
             final long seqNo = operation.seqNo();
-            if (startingSeqNo >= 0 && (seqNo == SequenceNumbers.UNASSIGNED_SEQ_NO || seqNo < startingSeqNo)) {
+            if (seqNo < startingSeqNo || seqNo > endingSeqNo) {
                 skippedOps++;
                 continue;
             }
@@ -552,6 +568,7 @@ public class RecoverySourceHandler {
             ops++;
             size += operation.estimateSize();
             totalSentOps++;
+            requiredOpsTracker.markSeqNoAsCompleted(seqNo);
 
             // check if this request is past bytes threshold, and if so, send it off
             if (size >= chunkSizeInBytes) {
@@ -569,8 +586,14 @@ public class RecoverySourceHandler {
         }
 
         assert expectedTotalOps == snapshot.overriddenOperations() + skippedOps + totalSentOps
-                : String.format(Locale.ROOT, "expected total [%d], overridden [%d], skipped [%d], total sent [%d]",
-                                    expectedTotalOps, snapshot.overriddenOperations(), skippedOps, totalSentOps);
+            : String.format(Locale.ROOT, "expected total [%d], overridden [%d], skipped [%d], total sent [%d]",
+            expectedTotalOps, snapshot.overriddenOperations(), skippedOps, totalSentOps);
+
+        if (requiredOpsTracker.getCheckpoint() < endingSeqNo) {
+            throw new IllegalStateException("translog replay failed to cover required sequence numbers" +
+                " (required range [" + requiredSeqNoRangeStart + ":" + endingSeqNo + "). first missing op is ["
+                + (requiredOpsTracker.getCheckpoint() + 1) + "]");
+        }
 
         logger.trace("sent final batch of [{}][{}] (total: [{}]) translog operations", ops, new ByteSizeValue(size), expectedTotalOps);
 
diff --git a/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
index 844d6b0aaf9..13e8d90eb47 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
@@ -374,15 +374,15 @@ public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestC
             IndexShard newReplica = shards.addReplicaWithExistingPath(replica.shardPath(), replica.routingEntry().currentNodeId());
 
             CountDownLatch recoveryStart = new CountDownLatch(1);
-            AtomicBoolean preparedForTranslog = new AtomicBoolean(false);
+            AtomicBoolean opsSent = new AtomicBoolean(false);
             final Future recoveryFuture = shards.asyncRecoverReplica(newReplica, (indexShard, node) -> {
                 recoveryStart.countDown();
                 return new RecoveryTarget(indexShard, node, recoveryListener, l -> {
                 }) {
                     @Override
-                    public void prepareForTranslogOperations(int totalTranslogOps) throws IOException {
-                        preparedForTranslog.set(true);
-                        super.prepareForTranslogOperations(totalTranslogOps);
+                    public long indexTranslogOperations(List operations, int totalTranslogOps) throws IOException {
+                        opsSent.set(true);
+                        return super.indexTranslogOperations(operations, totalTranslogOps);
                     }
                 };
             });
@@ -392,7 +392,7 @@ public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestC
             // index some more
             docs += shards.indexDocs(randomInt(5));
 
-            assertFalse("recovery should wait on pending docs", preparedForTranslog.get());
+            assertFalse("recovery should wait on pending docs", opsSent.get());
 
             primaryEngineFactory.releaseLatchedIndexers();
             pendingDocsDone.await();
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
index 993cc845064..c04d69bbed2 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
@@ -70,15 +70,18 @@ import org.elasticsearch.test.CorruptionUtils;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.mockito.ArgumentCaptor;
 
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
@@ -88,6 +91,7 @@ import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class RecoverySourceHandlerTests extends ESTestCase {
@@ -181,29 +185,68 @@ public class RecoverySourceHandlerTests extends ESTestCase {
             operations.add(new Translog.Index(index, new Engine.IndexResult(1, i - initialNumberOfDocs, true)));
         }
         operations.add(null);
-        final long startingSeqNo = randomBoolean() ? SequenceNumbers.UNASSIGNED_SEQ_NO : randomIntBetween(0, 16);
-        RecoverySourceHandler.SendSnapshotResult result = handler.sendSnapshot(startingSeqNo, new Translog.Snapshot() {
-            @Override
-            public void close() {
+        final long startingSeqNo = randomIntBetween(0, numberOfDocsWithValidSequenceNumbers - 1);
+        final long requiredStartingSeqNo = randomIntBetween((int) startingSeqNo, numberOfDocsWithValidSequenceNumbers - 1);
+        final long endingSeqNo = randomIntBetween((int) requiredStartingSeqNo - 1, numberOfDocsWithValidSequenceNumbers - 1);
+        RecoverySourceHandler.SendSnapshotResult result = handler.sendSnapshot(startingSeqNo, requiredStartingSeqNo,
+            endingSeqNo, new Translog.Snapshot() {
+                @Override
+                public void close() {
 
-            }
+                }
 
-            private int counter = 0;
+                private int counter = 0;
 
-            @Override
-            public int totalOperations() {
-                return operations.size() - 1;
-            }
+                @Override
+                public int totalOperations() {
+                    return operations.size() - 1;
+                }
 
-            @Override
-            public Translog.Operation next() throws IOException {
-                return operations.get(counter++);
-            }
-        });
-        if (startingSeqNo == SequenceNumbers.UNASSIGNED_SEQ_NO) {
-            assertThat(result.totalOperations, equalTo(initialNumberOfDocs + numberOfDocsWithValidSequenceNumbers));
-        } else {
-            assertThat(result.totalOperations, equalTo(Math.toIntExact(numberOfDocsWithValidSequenceNumbers - startingSeqNo)));
+                @Override
+                public Translog.Operation next() throws IOException {
+                    return operations.get(counter++);
+                }
+            });
+        final int expectedOps = (int) (endingSeqNo - startingSeqNo + 1);
+        assertThat(result.totalOperations, equalTo(expectedOps));
+        final ArgumentCaptor shippedOpsCaptor = ArgumentCaptor.forClass(List.class);
+        verify(recoveryTarget).indexTranslogOperations(shippedOpsCaptor.capture(), ArgumentCaptor.forClass(Integer.class).capture());
+        List shippedOps = shippedOpsCaptor.getAllValues().stream()
+            .flatMap(List::stream).map(o -> (Translog.Operation) o).collect(Collectors.toList());
+        shippedOps.sort(Comparator.comparing(Translog.Operation::seqNo));
+        assertThat(shippedOps.size(), equalTo(expectedOps));
+        for (int i = 0; i < shippedOps.size(); i++) {
+            assertThat(shippedOps.get(i), equalTo(operations.get(i + (int) startingSeqNo + initialNumberOfDocs)));
+        }
+        if (endingSeqNo >= requiredStartingSeqNo + 1) {
+            // check that missing ops blows up
+            List requiredOps = operations.subList(0, operations.size() - 1).stream() // remove last null marker
+                .filter(o -> o.seqNo() >= requiredStartingSeqNo && o.seqNo() <= endingSeqNo).collect(Collectors.toList());
+            List opsToSkip = randomSubsetOf(randomIntBetween(1, requiredOps.size()), requiredOps);
+            expectThrows(IllegalStateException.class, () ->
+                handler.sendSnapshot(startingSeqNo, requiredStartingSeqNo,
+                    endingSeqNo, new Translog.Snapshot() {
+                        @Override
+                        public void close() {
+
+                        }
+
+                        private int counter = 0;
+
+                        @Override
+                        public int totalOperations() {
+                            return operations.size() - 1 - opsToSkip.size();
+                        }
+
+                        @Override
+                        public Translog.Operation next() throws IOException {
+                            Translog.Operation op;
+                            do {
+                                op = operations.get(counter++);
+                            } while (op != null && opsToSkip.contains(op));
+                            return op;
+                        }
+                    }));
         }
     }
 
@@ -383,7 +426,7 @@ public class RecoverySourceHandlerTests extends ESTestCase {
             }
 
             @Override
-            long phase2(long startingSeqNo, Translog.Snapshot snapshot) throws IOException {
+            long phase2(long startingSeqNo, long requiredSeqNoRangeStart, long endingSeqNo, Translog.Snapshot snapshot) throws IOException {
                 phase2Called.set(true);
                 return SequenceNumbers.UNASSIGNED_SEQ_NO;
             }
diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
index 22859859f25..00d700d2d52 100644
--- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -26,8 +26,10 @@ import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -53,6 +55,8 @@ import java.util.regex.Pattern;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
+import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
+import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -585,6 +589,28 @@ public class FullClusterRestartIT extends ESRestTestCase {
         assertThat(toStr(client().performRequest("GET", docLocation)), containsString(doc));
     }
 
+    /**
+     * Tests that a single empty shard index is correctly recovered. Empty shards are often an edge case.
+     */
+    public void testEmptyShard() throws IOException {
+        final String index = "test_empty_shard";
+
+        if (runningAgainstOldCluster) {
+            Settings.Builder settings = Settings.builder()
+                .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+                .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                // if the node with the replica is the first to be restarted, while a replica is still recovering
+                // then delayed allocation will kick in. When the node comes back, the master will search for a copy
+                // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
+                // before timing out
+                .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
+                .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
+            createIndex(index, settings.build());
+        }
+        ensureGreen(index);
+    }
+
+
     /**
      * Tests recovery of an index with or without a translog and the
      * statistics we gather about that.
diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java
index 9de8954c531..958cd9fe3ef 100644
--- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java
+++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java
@@ -25,7 +25,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.seqno.SeqNoStats;
@@ -49,14 +48,6 @@ import static org.hamcrest.Matchers.not;
 
 public class IndexingIT extends ESRestTestCase {
 
-    private void updateIndexSetting(String name, Settings.Builder settings) throws IOException {
-        updateIndexSetting(name, settings.build());
-    }
-    private void updateIndexSetting(String name, Settings settings) throws IOException {
-        assertOK(client().performRequest("PUT", name + "/_settings", Collections.emptyMap(),
-            new StringEntity(Strings.toString(settings), ContentType.APPLICATION_JSON)));
-    }
-
     private int indexDocs(String index, final int idStart, final int numDocs) throws IOException {
         for (int i = 0; i < numDocs; i++) {
             final int id = idStart + i;
@@ -113,7 +104,7 @@ public class IndexingIT extends ESRestTestCase {
             final int finalVersionForDoc1 = indexDocWithConcurrentUpdates(index, 1, nUpdates);
             logger.info("allowing shards on all nodes");
             updateIndexSetting(index, Settings.builder().putNull("index.routing.allocation.include._name"));
-            ensureGreen();
+            ensureGreen(index);
             assertOK(client().performRequest("POST", index + "/_refresh"));
             List shards = buildShards(index, nodes, newNodeClient);
             Shard primary = buildShards(index, nodes, newNodeClient).stream().filter(Shard::isPrimary).findFirst().get();
@@ -138,7 +129,7 @@ public class IndexingIT extends ESRestTestCase {
             primary = shards.stream().filter(Shard::isPrimary).findFirst().get();
             logger.info("moving primary to new node by excluding {}", primary.getNode().getNodeName());
             updateIndexSetting(index, Settings.builder().put("index.routing.allocation.exclude._name", primary.getNode().getNodeName()));
-            ensureGreen();
+            ensureGreen(index);
             nUpdates = randomIntBetween(minUpdates, maxUpdates);
             logger.info("indexing docs with [{}] concurrent updates after moving primary", nUpdates);
             final int finalVersionForDoc3 = indexDocWithConcurrentUpdates(index, 3, nUpdates);
@@ -151,7 +142,7 @@ public class IndexingIT extends ESRestTestCase {
 
             logger.info("setting number of replicas to 0");
             updateIndexSetting(index, Settings.builder().put("index.number_of_replicas", 0));
-            ensureGreen();
+            ensureGreen(index);
             nUpdates = randomIntBetween(minUpdates, maxUpdates);
             logger.info("indexing doc with [{}] concurrent updates after setting number of replicas to 0", nUpdates);
             final int finalVersionForDoc4 = indexDocWithConcurrentUpdates(index, 4, nUpdates);
@@ -164,7 +155,7 @@ public class IndexingIT extends ESRestTestCase {
 
             logger.info("setting number of replicas to 1");
             updateIndexSetting(index, Settings.builder().put("index.number_of_replicas", 1));
-            ensureGreen();
+            ensureGreen(index);
             nUpdates = randomIntBetween(minUpdates, maxUpdates);
             logger.info("indexing doc with [{}] concurrent updates after setting number of replicas to 1", nUpdates);
             final int finalVersionForDoc5 = indexDocWithConcurrentUpdates(index, 5, nUpdates);
@@ -199,7 +190,7 @@ public class IndexingIT extends ESRestTestCase {
             assertSeqNoOnShards(index, nodes, nodes.getBWCVersion().major >= 6 ? numDocs : 0, newNodeClient);
             logger.info("allowing shards on all nodes");
             updateIndexSetting(index, Settings.builder().putNull("index.routing.allocation.include._name"));
-            ensureGreen();
+            ensureGreen(index);
             assertOK(client().performRequest("POST", index + "/_refresh"));
             for (final String bwcName : bwcNamesList) {
                 assertCount(index, "_only_nodes:" + bwcName, numDocs);
@@ -211,7 +202,7 @@ public class IndexingIT extends ESRestTestCase {
             Shard primary = buildShards(index, nodes, newNodeClient).stream().filter(Shard::isPrimary).findFirst().get();
             logger.info("moving primary to new node by excluding {}", primary.getNode().getNodeName());
             updateIndexSetting(index, Settings.builder().put("index.routing.allocation.exclude._name", primary.getNode().getNodeName()));
-            ensureGreen();
+            ensureGreen(index);
             int numDocsOnNewPrimary = 0;
             final int numberOfDocsAfterMovingPrimary = 1 + randomInt(5);
             logger.info("indexing [{}] docs after moving primary", numberOfDocsAfterMovingPrimary);
@@ -230,7 +221,7 @@ public class IndexingIT extends ESRestTestCase {
             numDocs += numberOfDocsAfterDroppingReplicas;
             logger.info("setting number of replicas to 1");
             updateIndexSetting(index, Settings.builder().put("index.number_of_replicas", 1));
-            ensureGreen();
+            ensureGreen(index);
             assertOK(client().performRequest("POST", index + "/_refresh"));
 
             for (Shard shard : buildShards(index, nodes, newNodeClient)) {
@@ -272,7 +263,7 @@ public class IndexingIT extends ESRestTestCase {
         final String index = "test-snapshot-index";
         createIndex(index, settings.build());
         indexDocs(index, 0, between(50, 100));
-        ensureGreen();
+        ensureGreen(index);
         assertOK(client().performRequest("POST", index + "/_refresh"));
 
         assertOK(
@@ -282,7 +273,7 @@ public class IndexingIT extends ESRestTestCase {
 
         // Allocating shards on all nodes, taking snapshots should happen on all nodes.
         updateIndexSetting(index, Settings.builder().putNull("index.routing.allocation.include._name"));
-        ensureGreen();
+        ensureGreen(index);
         assertOK(client().performRequest("POST", index + "/_refresh"));
 
         assertOK(
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
index 6c21d3e37ef..7869eb8a39d 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
@@ -18,16 +18,30 @@
  */
 package org.elasticsearch.upgrades;
 
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Future;
+import java.util.function.Predicate;
 
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength;
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
+import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING;
+import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.notNullValue;
@@ -89,7 +103,7 @@ public class RecoveryIT extends ESRestTestCase {
                 .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms");
             createIndex(index, settings.build());
         } else if (clusterType == CLUSTER_TYPE.UPGRADED) {
-            ensureGreen();
+            ensureGreen(index);
             Response response = client().performRequest("GET", index + "/_stats", Collections.singletonMap("level", "shards"));
             assertOK(response);
             ObjectPath objectPath = ObjectPath.createFromResponse(response);
@@ -109,4 +123,156 @@ public class RecoveryIT extends ESRestTestCase {
         }
     }
 
+    private int indexDocs(String index, final int idStart, final int numDocs) throws IOException {
+        for (int i = 0; i < numDocs; i++) {
+            final int id = idStart + i;
+            assertOK(client().performRequest("PUT", index + "/test/" + id, emptyMap(),
+                new StringEntity("{\"test\": \"test_" + randomAsciiOfLength(2) + "\"}", ContentType.APPLICATION_JSON)));
+        }
+        return numDocs;
+    }
+
+    private Future asyncIndexDocs(String index, final int idStart, final int numDocs) throws IOException {
+        PlainActionFuture future = new PlainActionFuture<>();
+        Thread background = new Thread(new AbstractRunnable() {
+            @Override
+            public void onFailure(Exception e) {
+                future.onFailure(e);
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+                indexDocs(index, idStart, numDocs);
+                future.onResponse(null);
+            }
+        });
+        background.start();
+        return future;
+    }
+
+    public void testRecoveryWithConcurrentIndexing() throws Exception {
+        final String index = "recovery_with_concurrent_indexing";
+        Response response = client().performRequest("GET", "_nodes");
+        ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        final Map nodeMap = objectPath.evaluate("nodes");
+        List nodes = new ArrayList<>(nodeMap.keySet());
+
+        switch (clusterType) {
+            case OLD:
+                Settings.Builder settings = Settings.builder()
+                    .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+                    .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                    // if the node with the replica is the first to be restarted, while a replica is still recovering
+                    // then delayed allocation will kick in. When the node comes back, the master will search for a copy
+                    // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
+                    // before timing out
+                    .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
+                    .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
+                createIndex(index, settings.build());
+                indexDocs(index, 0, 10);
+                ensureGreen(index);
+                // make sure that we can index while the replicas are recovering
+                updateIndexSetting(index, Settings.builder().put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "primaries"));
+                break;
+            case MIXED:
+                updateIndexSetting(index, Settings.builder().put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), (String)null));
+                asyncIndexDocs(index, 10, 50).get();
+                ensureGreen(index);
+                assertOK(client().performRequest("POST", index + "/_refresh"));
+                assertCount(index, "_only_nodes:" + nodes.get(0), 60);
+                assertCount(index, "_only_nodes:" + nodes.get(1), 60);
+                // make sure that we can index while the replicas are recovering
+                updateIndexSetting(index, Settings.builder().put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "primaries"));
+                break;
+            case UPGRADED:
+                updateIndexSetting(index, Settings.builder().put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), (String)null));
+                asyncIndexDocs(index, 60, 50).get();
+                ensureGreen(index);
+                assertOK(client().performRequest("POST", index + "/_refresh"));
+                assertCount(index, "_only_nodes:" + nodes.get(0), 60);
+                assertCount(index, "_only_nodes:" + nodes.get(1), 60);
+                break;
+            default:
+                throw new IllegalStateException("unknown type " + clusterType);
+        }
+    }
+
+    private void assertCount(final String index, final String preference, final int expectedCount) throws IOException {
+        final Response response = client().performRequest("GET", index + "/_count", Collections.singletonMap("preference", preference));
+        assertOK(response);
+        final int actualCount = Integer.parseInt(ObjectPath.createFromResponse(response).evaluate("count").toString());
+        assertThat(actualCount, equalTo(expectedCount));
+    }
+
+
+    private String getNodeId(Predicate versionPredicate) throws IOException {
+        Response response = client().performRequest("GET", "_nodes");
+        ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        Map nodesAsMap = objectPath.evaluate("nodes");
+        for (String id : nodesAsMap.keySet()) {
+            Version version = Version.fromString(objectPath.evaluate("nodes." + id + ".version"));
+            if (versionPredicate.test(version)) {
+                return id;
+            }
+        }
+        return null;
+    }
+
+
+    public void testRelocationWithConcurrentIndexing() throws Exception {
+        final String index = "relocation_with_concurrent_indexing";
+        switch (clusterType) {
+            case OLD:
+                Settings.Builder settings = Settings.builder()
+                    .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+                    .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                    // if the node with the replica is the first to be restarted, while a replica is still recovering
+                    // then delayed allocation will kick in. When the node comes back, the master will search for a copy
+                    // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
+                    // before timing out
+                    .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
+                    .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
+                createIndex(index, settings.build());
+                indexDocs(index, 0, 10);
+                ensureGreen(index);
+                // make sure that no shards are allocated, so we can make sure the primary stays on the old node (when one
+                // node stops, we lose the master too, so a replica will not be promoted)
+                updateIndexSetting(index, Settings.builder().put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none"));
+                break;
+            case MIXED:
+                final String newNode = getNodeId(v -> v.equals(Version.CURRENT));
+                final String oldNode = getNodeId(v -> v.before(Version.CURRENT));
+                // remove the replica now that we know that the primary is an old node
+                updateIndexSetting(index, Settings.builder()
+                    .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+                    .put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), (String)null)
+                    .put("index.routing.allocation.include._id", oldNode)
+                );
+                updateIndexSetting(index, Settings.builder().put("index.routing.allocation.include._id", newNode));
+                asyncIndexDocs(index, 10, 50).get();
+                ensureGreen(index);
+                assertOK(client().performRequest("POST", index + "/_refresh"));
+                assertCount(index, "_only_nodes:" + newNode, 60);
+                break;
+            case UPGRADED:
+                updateIndexSetting(index, Settings.builder()
+                    .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                    .put("index.routing.allocation.include._id", (String)null)
+                );
+                asyncIndexDocs(index, 60, 50).get();
+                ensureGreen(index);
+                assertOK(client().performRequest("POST", index + "/_refresh"));
+                Response response = client().performRequest("GET", "_nodes");
+                ObjectPath objectPath = ObjectPath.createFromResponse(response);
+                final Map nodeMap = objectPath.evaluate("nodes");
+                List nodes = new ArrayList<>(nodeMap.keySet());
+
+                assertCount(index, "_only_nodes:" + nodes.get(0), 110);
+                assertCount(index, "_only_nodes:" + nodes.get(1), 110);
+                break;
+            default:
+                throw new IllegalStateException("unknown type " + clusterType);
+        }
+    }
+
 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml
index b24025f356c..29790e21461 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml
@@ -15,6 +15,16 @@
           # allocation will kick in, and the cluster health won't return to GREEN
           # before timing out
           index.unassigned.node_left.delayed_timeout: "100ms"
+
+  - do:
+      indices.create:
+        index: empty_index # index to ensure we can recover empty indices
+        body:
+          # if the node with the replica is the first to be restarted, then delayed
+          # allocation will kick in, and the cluster health won't return to GREEN
+          # before timing out
+          index.unassigned.node_left.delayed_timeout: "100ms"
+
   - do:
       bulk:
         refresh: true
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
index cdc94f638b5..af58a2f362c 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
@@ -7,6 +7,7 @@
         # wait for long enough that we give delayed unassigned shards to stop being delayed
         timeout: 70s
         level: shards
+        index: test_index,index_with_replicas,empty_index
 
  - do:
      search:
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index cef820ac009..0da1aff2ef7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -392,13 +392,18 @@ public abstract class ESRestTestCase extends ESTestCase {
         assertThat(response.getStatusLine().getStatusCode(), anyOf(equalTo(200), equalTo(201)));
     }
 
-    protected void ensureGreen() throws IOException {
+    /**
+     * checks that the specific index is green. we force a selection of an index as the tests share a cluster and often leave indices
+     * in an non green state
+     * @param index index to test for
+     **/
+    protected void ensureGreen(String index) throws IOException {
         Map params = new HashMap<>();
         params.put("wait_for_status", "green");
         params.put("wait_for_no_relocating_shards", "true");
         params.put("timeout", "70s");
         params.put("level", "shards");
-        assertOK(client().performRequest("GET", "_cluster/health", params));
+        assertOK(client().performRequest("GET", "_cluster/health/" + index, params));
     }
 
     protected void createIndex(String name, Settings settings) throws IOException {
@@ -411,4 +416,12 @@ public abstract class ESRestTestCase extends ESTestCase {
                 + ", \"mappings\" : {" + mapping + "} }", ContentType.APPLICATION_JSON)));
     }
 
+    protected void updateIndexSetting(String index, Settings.Builder settings) throws IOException {
+        updateIndexSetting(index, settings.build());
+    }
+
+    private void updateIndexSetting(String index, Settings settings) throws IOException {
+        assertOK(client().performRequest("PUT", index + "/_settings", Collections.emptyMap(),
+            new StringEntity(Strings.toString(settings), ContentType.APPLICATION_JSON)));
+    }
 }

From f58a3d0b96bf33ebd743d53062b62552833faa49 Mon Sep 17 00:00:00 2001
From: Boaz Leskes 
Date: Fri, 1 Dec 2017 09:02:03 +0100
Subject: [PATCH 159/297] testRelocationWithConcurrentIndexing: wait for green
 (on relevan index) and shard initialization to settle down before starting
 relocation

---
 .../java/org/elasticsearch/upgrades/RecoveryIT.java  |  4 +++-
 .../org/elasticsearch/test/rest/ESRestTestCase.java  | 12 ++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
index 7869eb8a39d..0cf91614bef 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
@@ -242,12 +242,14 @@ public class RecoveryIT extends ESRestTestCase {
             case MIXED:
                 final String newNode = getNodeId(v -> v.equals(Version.CURRENT));
                 final String oldNode = getNodeId(v -> v.before(Version.CURRENT));
-                // remove the replica now that we know that the primary is an old node
+                // remove the replica and guaranteed the primary is placed on the old node
                 updateIndexSetting(index, Settings.builder()
                     .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
                     .put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), (String)null)
                     .put("index.routing.allocation.include._id", oldNode)
                 );
+                ensureGreen(index); // wait for the primary to be assigned
+                ensureNoInitializingShards(); // wait for all other shard activity to finish
                 updateIndexSetting(index, Settings.builder().put("index.routing.allocation.include._id", newNode));
                 asyncIndexDocs(index, 10, 50).get();
                 ensureGreen(index);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 0da1aff2ef7..f1a63c926fe 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -406,6 +406,18 @@ public abstract class ESRestTestCase extends ESTestCase {
         assertOK(client().performRequest("GET", "_cluster/health/" + index, params));
     }
 
+    /**
+     * waits until all shard initialization is completed. This is a handy alternative to ensureGreen as it relates to all shards
+     * in the cluster and doesn't require to know how many nodes/replica there are.
+     */
+    protected void ensureNoInitializingShards() throws IOException {
+        Map params = new HashMap<>();
+        params.put("wait_for_no_initializing_shards", "true");
+        params.put("timeout", "70s");
+        params.put("level", "shards");
+        assertOK(client().performRequest("GET", "_cluster/health/", params));
+    }
+
     protected void createIndex(String name, Settings settings) throws IOException {
         createIndex(name, settings, "");
     }

From 2900e3f34528b21215ca736f65ae7fd5d75c5c80 Mon Sep 17 00:00:00 2001
From: Boaz Leskes 
Date: Sun, 3 Dec 2017 15:36:44 +0100
Subject: [PATCH 160/297] adapt testWaitForPendingSeqNo to stricter operation
 recovery range

Before we use to ship anything in the translog above a certain point. #27580 changed to have a strict upper bound.
---
 .../replication/RecoveryDuringReplicationTests.java      | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
index 13e8d90eb47..f4e646030f2 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
@@ -58,10 +58,12 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 
 public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestCase {
@@ -390,7 +392,8 @@ public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestC
             recoveryStart.await();
 
             // index some more
-            docs += shards.indexDocs(randomInt(5));
+            final int indexedDuringRecovery = shards.indexDocs(randomInt(5));
+            docs += indexedDuringRecovery;
 
             assertFalse("recovery should wait on pending docs", opsSent.get());
 
@@ -401,7 +404,9 @@ public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestC
             recoveryFuture.get();
 
             assertThat(newReplica.recoveryState().getIndex().fileDetails(), empty());
-            assertThat(newReplica.recoveryState().getTranslog().recoveredOperations(), equalTo(docs));
+            assertThat(newReplica.recoveryState().getTranslog().recoveredOperations(),
+                // we don't know which of the inflight operations made it into the translog range we re-play
+                both(greaterThanOrEqualTo(docs-indexedDuringRecovery)).and(lessThanOrEqualTo(docs)));
 
             shards.assertAllEqual(docs);
         } finally {

From e213fa033d2d1baebfe568c2722d58b71d1590b4 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen 
Date: Mon, 4 Dec 2017 09:51:34 -0500
Subject: [PATCH 161/297] Tighten the CountedBitSet class

This commit addresses the missed comments from https://github.com/elastic/elasticsearch/pull/27547.
---
 .../index/translog/CountedBitSet.java         | 20 ++++++++++---------
 .../index/translog/CountedBitSetTests.java    |  5 +++++
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/translog/CountedBitSet.java b/core/src/main/java/org/elasticsearch/index/translog/CountedBitSet.java
index 9fac230c9a8..ca1ae279a99 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/CountedBitSet.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/CountedBitSet.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.translog;
 
 import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * A {@link CountedBitSet} wraps a {@link FixedBitSet} but automatically releases the internal bitset
@@ -28,11 +29,14 @@ import org.apache.lucene.util.FixedBitSet;
  * from translog as these numbers are likely to form contiguous ranges (eg. filling all bits).
  */
 final class CountedBitSet extends BitSet {
+    static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(CountedBitSet.class);
     private short onBits; // Number of bits are set.
     private FixedBitSet bitset;
 
     CountedBitSet(short numBits) {
-        assert numBits > 0;
+        if (numBits <= 0) {
+            throw new IllegalArgumentException("Number of bits must be positive. Given [" + numBits + "]");
+        }
         this.onBits = 0;
         this.bitset = new FixedBitSet(numBits);
     }
@@ -41,7 +45,6 @@ final class CountedBitSet extends BitSet {
     public boolean get(int index) {
         assert 0 <= index && index < this.length();
         assert bitset == null || onBits < bitset.length() : "Bitset should be released when all bits are set";
-
         return bitset == null ? true : bitset.get(index);
     }
 
@@ -52,7 +55,7 @@ final class CountedBitSet extends BitSet {
 
         // Ignore set when bitset is full.
         if (bitset != null) {
-            boolean wasOn = bitset.getAndSet(index);
+            final boolean wasOn = bitset.getAndSet(index);
             if (wasOn == false) {
                 onBits++;
                 // Once all bits are set, we can simply just return YES for all indexes.
@@ -66,12 +69,12 @@ final class CountedBitSet extends BitSet {
 
     @Override
     public void clear(int startIndex, int endIndex) {
-        throw new UnsupportedOperationException("Not implemented yet");
+        throw new UnsupportedOperationException();
     }
 
     @Override
     public void clear(int index) {
-        throw new UnsupportedOperationException("Not implemented yet");
+        throw new UnsupportedOperationException();
     }
 
     @Override
@@ -86,20 +89,19 @@ final class CountedBitSet extends BitSet {
 
     @Override
     public int prevSetBit(int index) {
-        throw new UnsupportedOperationException("Not implemented yet");
+        throw new UnsupportedOperationException();
     }
 
     @Override
     public int nextSetBit(int index) {
-        throw new UnsupportedOperationException("Not implemented yet");
+        throw new UnsupportedOperationException();
     }
 
     @Override
     public long ramBytesUsed() {
-        throw new UnsupportedOperationException("Not implemented yet");
+        return BASE_RAM_BYTES_USED + (bitset == null ? 0 : bitset.ramBytesUsed());
     }
 
-    // Exposed for testing
     boolean isInternalBitsetReleased() {
         return bitset == null;
     }
diff --git a/core/src/test/java/org/elasticsearch/index/translog/CountedBitSetTests.java b/core/src/test/java/org/elasticsearch/index/translog/CountedBitSetTests.java
index 5174d1755be..b68607f02d6 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/CountedBitSetTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/CountedBitSetTests.java
@@ -26,7 +26,9 @@ import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.lessThan;
 
 public class CountedBitSetTests extends ESTestCase {
 
@@ -53,6 +55,7 @@ public class CountedBitSetTests extends ESTestCase {
         int numBits = (short) randomIntBetween(8, 4096);
         final CountedBitSet countedBitSet = new CountedBitSet((short) numBits);
         final List values = IntStream.range(0, numBits).boxed().collect(Collectors.toList());
+        final long ramBytesUsedWithBitSet = countedBitSet.ramBytesUsed();
 
         for (int i = 1; i < numBits; i++) {
             final int value = values.get(i);
@@ -65,6 +68,7 @@ public class CountedBitSetTests extends ESTestCase {
             assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(false));
             assertThat(countedBitSet.length(), equalTo(numBits));
             assertThat(countedBitSet.cardinality(), equalTo(i));
+            assertThat(countedBitSet.ramBytesUsed(), equalTo(ramBytesUsedWithBitSet));
         }
 
         // The missing piece to fill all bits.
@@ -79,6 +83,7 @@ public class CountedBitSetTests extends ESTestCase {
             assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(true));
             assertThat(countedBitSet.length(), equalTo(numBits));
             assertThat(countedBitSet.cardinality(), equalTo(numBits));
+            assertThat(countedBitSet.ramBytesUsed(), allOf(equalTo(CountedBitSet.BASE_RAM_BYTES_USED), lessThan(ramBytesUsedWithBitSet)));
         }
 
         // Tests with released internal bitset.

From 84ec47242807f5449edae60591e12c4fd9f55117 Mon Sep 17 00:00:00 2001
From: Simon Willnauer 
Date: Mon, 4 Dec 2017 16:33:47 +0100
Subject: [PATCH 162/297] Include internal refreshes in refresh stats (#27615)

Today we exclude internal refreshes in the refresh stats. Yet, it's very much
confusing to not take these into account. This change includes internal refreshes
into the stats until we have a dedicated stats for this.
---
 .../index/engine/EngineConfig.java            | 22 ++++++++++++++-----
 .../index/engine/InternalEngine.java          |  5 ++++-
 .../elasticsearch/index/shard/IndexShard.java |  5 +++--
 .../index/engine/InternalEngineTests.java     | 11 +++++-----
 .../index/shard/IndexShardTests.java          | 22 +++++--------------
 .../index/shard/RefreshListenersTests.java    |  4 ++--
 .../index/engine/EngineTestCase.java          |  9 ++++----
 7 files changed, 42 insertions(+), 36 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
index f923abc1a6c..8c134b140bd 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
@@ -69,7 +69,9 @@ public final class EngineConfig {
     private final QueryCache queryCache;
     private final QueryCachingPolicy queryCachingPolicy;
     @Nullable
-    private final List refreshListeners;
+    private final List externalRefreshListener;
+    @Nullable
+    private final List internalRefreshListener;
     @Nullable
     private final Sort indexSort;
     private final boolean forceNewHistoryUUID;
@@ -120,7 +122,8 @@ public final class EngineConfig {
                         Similarity similarity, CodecService codecService, Engine.EventListener eventListener,
                         QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
                         boolean forceNewHistoryUUID, TranslogConfig translogConfig, TimeValue flushMergesAfter,
-                        List refreshListeners, Sort indexSort,
+                        List externalRefreshListener,
+                        List internalRefreshListener, Sort indexSort,
                         TranslogRecoveryRunner translogRecoveryRunner, CircuitBreakerService circuitBreakerService) {
         if (openMode == null) {
             throw new IllegalArgumentException("openMode must not be null");
@@ -147,7 +150,8 @@ public final class EngineConfig {
         this.flushMergesAfter = flushMergesAfter;
         this.openMode = openMode;
         this.forceNewHistoryUUID = forceNewHistoryUUID;
-        this.refreshListeners = refreshListeners;
+        this.externalRefreshListener = externalRefreshListener;
+        this.internalRefreshListener = internalRefreshListener;
         this.indexSort = indexSort;
         this.translogRecoveryRunner = translogRecoveryRunner;
         this.circuitBreakerService = circuitBreakerService;
@@ -343,12 +347,18 @@ public final class EngineConfig {
     }
 
     /**
-     * The refresh listeners to add to Lucene
+     * The refresh listeners to add to Lucene for externally visible refreshes
      */
-    public List getRefreshListeners() {
-        return refreshListeners;
+    public List getExternalRefreshListener() {
+        return externalRefreshListener;
     }
 
+    /**
+     * The refresh listeners to add to Lucene for internally visible refreshes. These listeners will also be invoked on external refreshes
+     */
+    public List getInternalRefreshListener() { return internalRefreshListener;}
+
+
     /**
      * returns true if the engine is allowed to optimize indexing operations with an auto-generated ID
      */
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index fe8a33f5ecd..53747b063df 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -232,9 +232,12 @@ public class InternalEngine extends Engine {
             assert pendingTranslogRecovery.get() == false : "translog recovery can't be pending before we set it";
             // don't allow commits until we are done with recovering
             pendingTranslogRecovery.set(openMode == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG);
-            for (ReferenceManager.RefreshListener listener: engineConfig.getRefreshListeners()) {
+            for (ReferenceManager.RefreshListener listener: engineConfig.getExternalRefreshListener()) {
                 this.externalSearcherManager.addListener(listener);
             }
+            for (ReferenceManager.RefreshListener listener: engineConfig.getInternalRefreshListener()) {
+                this.internalSearcherManager.addListener(listener);
+            }
             success = true;
         } finally {
             if (success == false) {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 12da974645c..f0246060acf 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -2194,8 +2194,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             mapperService.indexAnalyzer(), similarityService.similarity(mapperService), codecService, shardEventListener,
             indexCache.query(), cachingPolicy, forceNewHistoryUUID, translogConfig,
             IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()),
-            Arrays.asList(refreshListeners, new RefreshMetricUpdater(refreshMetric)), indexSort,
-            this::runTranslogRecovery, circuitBreakerService);
+            Collections.singletonList(refreshListeners),
+            Collections.singletonList(new RefreshMetricUpdater(refreshMetric)),
+            indexSort, this::runTranslogRecovery, circuitBreakerService);
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 026a01a23c3..1b700b80086 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -105,7 +105,6 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.RootObjectMapper;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
-import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
@@ -2547,8 +2546,8 @@ public class InternalEngineTests extends EngineTestCase {
                 threadPool, config.getIndexSettings(), null, store, newMergePolicy(), config.getAnalyzer(), config.getSimilarity(),
                 new CodecService(null, logger), config.getEventListener(), IndexSearcher.getDefaultQueryCache(),
                 IndexSearcher.getDefaultQueryCachingPolicy(), false, translogConfig, TimeValue.timeValueMinutes(5),
-                config.getRefreshListeners(), null, config.getTranslogRecoveryRunner(), new NoneCircuitBreakerService());
-
+                config.getExternalRefreshListener(), config.getInternalRefreshListener(), null, config.getTranslogRecoveryRunner(),
+                new NoneCircuitBreakerService());
         try {
             InternalEngine internalEngine = new InternalEngine(brokenConfig);
             fail("translog belongs to a different engine");
@@ -2601,7 +2600,8 @@ public class InternalEngineTests extends EngineTestCase {
             threadPool, indexSettings, null, store, newMergePolicy(), config.getAnalyzer(), config.getSimilarity(),
             new CodecService(null, logger), config.getEventListener(), IndexSearcher.getDefaultQueryCache(),
             IndexSearcher.getDefaultQueryCachingPolicy(), false, config.getTranslogConfig(), TimeValue.timeValueMinutes(5),
-            config.getRefreshListeners(), null, config.getTranslogRecoveryRunner(), new NoneCircuitBreakerService());
+            config.getExternalRefreshListener(), config.getInternalRefreshListener(), null, config.getTranslogRecoveryRunner(),
+            new NoneCircuitBreakerService());
         engine = new InternalEngine(newConfig);
         if (newConfig.getOpenMode() == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG) {
             engine.recoverFromTranslog();
@@ -2631,7 +2631,8 @@ public class InternalEngineTests extends EngineTestCase {
             threadPool, config.getIndexSettings(), null, store, newMergePolicy(), config.getAnalyzer(), config.getSimilarity(),
             new CodecService(null, logger), config.getEventListener(), IndexSearcher.getDefaultQueryCache(),
             IndexSearcher.getDefaultQueryCachingPolicy(), true, config.getTranslogConfig(), TimeValue.timeValueMinutes(5),
-            config.getRefreshListeners(), null, config.getTranslogRecoveryRunner(), new NoneCircuitBreakerService());
+            config.getExternalRefreshListener(), config.getInternalRefreshListener(), null, config.getTranslogRecoveryRunner(),
+            new NoneCircuitBreakerService());
         engine = new InternalEngine(newConfig);
         if (newConfig.getOpenMode() == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG) {
             engine.recoverFromTranslog();
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 96bcb9382ee..dc4294f30f5 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -92,6 +92,7 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.index.store.Store;
@@ -1215,26 +1216,15 @@ public class IndexShardTests extends IndexShardTestCase {
         long refreshCount = shard.refreshStats().getTotal();
         indexDoc(shard, "test", "test");
         try (Engine.GetResult ignored = shard.get(new Engine.Get(true, "test", "test",
-                new Term(IdFieldMapper.NAME, Uid.encodeId("test"))))) {
-            assertThat(shard.refreshStats().getTotal(), equalTo(refreshCount));
+            new Term(IdFieldMapper.NAME, Uid.encodeId("test"))))) {
+            assertThat(shard.refreshStats().getTotal(), equalTo(refreshCount+1));
         }
+        indexDoc(shard, "test", "test");
+        shard.writeIndexingBuffer();
+        assertThat(shard.refreshStats().getTotal(), equalTo(refreshCount+2));
         closeShards(shard);
     }
 
-    private ParsedDocument testParsedDocument(String id, String type, String routing,
-                                              ParseContext.Document document, BytesReference source, Mapping mappingUpdate) {
-        Field idField = new Field("_id", id, IdFieldMapper.Defaults.FIELD_TYPE);
-        Field versionField = new NumericDocValuesField("_version", 0);
-        SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
-        document.add(idField);
-        document.add(versionField);
-        document.add(seqID.seqNo);
-        document.add(seqID.seqNoDocValue);
-        document.add(seqID.primaryTerm);
-        return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, XContentType.JSON,
-            mappingUpdate);
-    }
-
     public void testIndexingOperationsListeners() throws IOException {
         IndexShard shard = newStartedShard(true);
         indexDoc(shard, "test", "0", "{\"foo\" : \"bar\"}");
diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index e3158a21853..125f45fd007 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -55,7 +55,6 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.Scheduler.Cancellable;
@@ -123,7 +122,8 @@ public class RefreshListenersTests extends ESTestCase {
         EngineConfig config = new EngineConfig(EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG, shardId, allocationId, threadPool,
                 indexSettings, null, store, newMergePolicy(), iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger),
                 eventListener, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), false, translogConfig,
-                TimeValue.timeValueMinutes(5), Collections.singletonList(listeners), null, null, new NoneCircuitBreakerService());
+                TimeValue.timeValueMinutes(5), Collections.singletonList(listeners), Collections.emptyList(), null, null,
+                new NoneCircuitBreakerService());
         engine = new InternalEngine(config);
         listeners.setTranslog(engine.getTranslog());
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
index 9ba6f64d74c..bfc44f71d80 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
@@ -162,8 +162,9 @@ public abstract class EngineTestCase extends ESTestCase {
         return new EngineConfig(openMode, config.getShardId(), config.getAllocationId(), config.getThreadPool(), config.getIndexSettings(),
                 config.getWarmer(), config.getStore(), config.getMergePolicy(), analyzer, config.getSimilarity(),
                 new CodecService(null, logger), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(),
-                config.getForceNewHistoryUUID(), config.getTranslogConfig(), config.getFlushMergesAfter(), config.getRefreshListeners(),
-                config.getIndexSort(), config.getTranslogRecoveryRunner(), config.getCircuitBreakerService());
+                config.getForceNewHistoryUUID(), config.getTranslogConfig(), config.getFlushMergesAfter(),
+                config.getExternalRefreshListener(), Collections.emptyList(), config.getIndexSort(), config.getTranslogRecoveryRunner(),
+                config.getCircuitBreakerService());
     }
 
     @Override
@@ -402,8 +403,8 @@ public abstract class EngineTestCase extends ESTestCase {
         EngineConfig config = new EngineConfig(openMode, shardId, allocationId.getId(), threadPool, indexSettings, null, store,
                 mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), listener,
                 IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), false, translogConfig,
-                TimeValue.timeValueMinutes(5), refreshListenerList, indexSort, handler, new NoneCircuitBreakerService());
-
+                TimeValue.timeValueMinutes(5), refreshListenerList, Collections.emptyList(), indexSort, handler,
+                new NoneCircuitBreakerService());
         return config;
     }
 

From 0b50b313d2f84a015c31355665ebb189afd3e6a4 Mon Sep 17 00:00:00 2001
From: Boaz Leskes 
Date: Mon, 4 Dec 2017 18:04:32 +0100
Subject: [PATCH 163/297] RecoveryIT.testRecoveryWithConcurrentIndexing should
 check for 110 docs in an upgraded cluster

Closes #27650
---
 .../src/test/java/org/elasticsearch/upgrades/RecoveryIT.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
index 0cf91614bef..d20ccc7e474 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
@@ -189,8 +189,8 @@ public class RecoveryIT extends ESRestTestCase {
                 asyncIndexDocs(index, 60, 50).get();
                 ensureGreen(index);
                 assertOK(client().performRequest("POST", index + "/_refresh"));
-                assertCount(index, "_only_nodes:" + nodes.get(0), 60);
-                assertCount(index, "_only_nodes:" + nodes.get(1), 60);
+                assertCount(index, "_only_nodes:" + nodes.get(0), 110);
+                assertCount(index, "_only_nodes:" + nodes.get(1), 110);
                 break;
             default:
                 throw new IllegalStateException("unknown type " + clusterType);

From 17a2d574de7ad503e7010618c01386f621034c90 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Mon, 4 Dec 2017 12:14:25 -0500
Subject: [PATCH 164/297] Obey translog durability in global checkpoint sync

After write operations in some situations we fire a post-operation
global checkpoint sync. The global checkpoint sync unconditionally
fsyncs the translog and this can then look like an fsync
per-request. This violates the translog durability settings on the index
if this durability is set to async. This commit changes the global
checkpoint sync to observe the translog durability.

Relates #27641
---
 .../index/seqno/GlobalCheckpointSyncAction.java        |  9 +++++++--
 .../index/seqno/GlobalCheckpointSyncActionTests.java   | 10 +++++++++-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java b/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
index 2c60ebfac6b..04caf187db9 100644
--- a/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
+++ b/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
@@ -39,6 +39,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -116,13 +117,17 @@ public class GlobalCheckpointSyncAction extends TransportReplicationAction<
     @Override
     protected PrimaryResult shardOperationOnPrimary(
             final Request request, final IndexShard indexShard) throws Exception {
-        indexShard.getTranslog().sync();
+        if (indexShard.getTranslogDurability() == Translog.Durability.REQUEST) {
+            indexShard.getTranslog().sync();
+        }
         return new PrimaryResult<>(request, new ReplicationResponse());
     }
 
     @Override
     protected ReplicaResult shardOperationOnReplica(final Request request, final IndexShard indexShard) throws Exception {
-        indexShard.getTranslog().sync();
+        if (indexShard.getTranslogDurability() == Translog.Durability.REQUEST) {
+            indexShard.getTranslog().sync();
+        }
         return new ReplicaResult();
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java b/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
index a03b506cba5..c327e47c30c 100644
--- a/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
+++ b/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
@@ -38,6 +38,7 @@ import org.elasticsearch.transport.TransportService;
 
 import java.util.Collections;
 
+import static org.elasticsearch.mock.orig.Mockito.never;
 import static org.elasticsearch.mock.orig.Mockito.when;
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.mockito.Mockito.mock;
@@ -86,6 +87,9 @@ public class GlobalCheckpointSyncActionTests extends ESTestCase {
         final ShardId shardId = new ShardId(index, id);
         when(indexShard.shardId()).thenReturn(shardId);
 
+        final Translog.Durability durability = randomFrom(Translog.Durability.ASYNC, Translog.Durability.REQUEST);
+        when(indexShard.getTranslogDurability()).thenReturn(durability);
+
         final Translog translog = mock(Translog.class);
         when(indexShard.getTranslog()).thenReturn(translog);
 
@@ -105,7 +109,11 @@ public class GlobalCheckpointSyncActionTests extends ESTestCase {
             action.shardOperationOnReplica(new GlobalCheckpointSyncAction.Request(indexShard.shardId()), indexShard);
         }
 
-        verify(translog).sync();
+        if (durability == Translog.Durability.ASYNC) {
+            verify(translog, never()).sync();
+        } else {
+            verify(translog).sync();
+        }
     }
 
 }

From da50fa4540f4dcbcf5496237a122b32881c34b24 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi 
Date: Mon, 4 Dec 2017 21:04:14 +0100
Subject: [PATCH 165/297] Fix highlighting on a keyword field that defines a
 normalizer (#27604)

* Fix highlighting on a keyword field that defines a normalizer

The `plain` and sometimes the `unified` highlighters need to re-analyze the content to highlight a field
This change makes sure that we don't ignore the normalizer defined on the keyword field for this analysis.
---
 .../subphase/highlight/PlainHighlighter.java  | 14 ++--
 .../highlight/UnifiedHighlighter.java         | 79 ++++++++-----------
 .../highlight/HighlighterSearchIT.java        | 28 +++++++
 3 files changed, 67 insertions(+), 54 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
index c7943367d31..23ae1e9154c 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
@@ -31,7 +31,6 @@ import org.apache.lucene.search.highlight.SimpleFragmenter;
 import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
 import org.apache.lucene.search.highlight.TextFragment;
-import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefHash;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ExceptionsHelper;
@@ -48,6 +47,9 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.search.fetch.subphase.highlight.UnifiedHighlighter.convertFieldValue;
+import static org.elasticsearch.search.fetch.subphase.highlight.UnifiedHighlighter.getAnalyzer;
+
 public class PlainHighlighter implements Highlighter {
     private static final String CACHE_KEY = "highlight-plain";
 
@@ -100,18 +102,12 @@ public class PlainHighlighter implements Highlighter {
         int numberOfFragments = field.fieldOptions().numberOfFragments() == 0 ? 1 : field.fieldOptions().numberOfFragments();
         ArrayList fragsList = new ArrayList<>();
         List textsToHighlight;
-        Analyzer analyzer = context.mapperService().documentMapper(hitContext.hit().getType()).mappers().indexAnalyzer();
-
+        Analyzer analyzer = getAnalyzer(context.mapperService().documentMapper(hitContext.hit().getType()), mapper.fieldType());
         try {
             textsToHighlight = HighlightUtils.loadFieldValues(field, mapper, context, hitContext);
 
             for (Object textToHighlight : textsToHighlight) {
-                String text;
-                if (textToHighlight instanceof BytesRef) {
-                    text = mapper.fieldType().valueForDisplay(textToHighlight).toString();
-                } else {
-                    text = textToHighlight.toString();
-                }
+                String text = convertFieldValue(mapper.fieldType(), textToHighlight);
 
                 try (TokenStream tokenStream = analyzer.tokenStream(mapper.fieldType().name(), text)) {
                     if (!tokenStream.hasAttribute(CharTermAttribute.class) || !tokenStream.hasAttribute(OffsetAttribute.class)) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
index 034cac9e5f9..06dd9232a74 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
@@ -32,8 +32,11 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.SearchContext;
@@ -50,8 +53,6 @@ import java.util.stream.Collectors;
 import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR;
 
 public class UnifiedHighlighter implements Highlighter {
-    private static final String CACHE_KEY = "highlight-unified";
-
     @Override
     public boolean canHighlight(FieldMapper fieldMapper) {
         return true;
@@ -63,36 +64,20 @@ public class UnifiedHighlighter implements Highlighter {
         SearchContextHighlight.Field field = highlighterContext.field;
         SearchContext context = highlighterContext.context;
         FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;
-
-        if (!hitContext.cache().containsKey(CACHE_KEY)) {
-            hitContext.cache().put(CACHE_KEY, new HighlighterEntry());
-        }
-
-        HighlighterEntry highlighterEntry = (HighlighterEntry) hitContext.cache().get(CACHE_KEY);
-        MapperHighlighterEntry mapperHighlighterEntry = highlighterEntry.mappers.get(fieldMapper);
-
-        if (mapperHighlighterEntry == null) {
-            Encoder encoder = field.fieldOptions().encoder().equals("html") ?
-                HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
-            CustomPassageFormatter passageFormatter =
-                new CustomPassageFormatter(field.fieldOptions().preTags()[0],
-                    field.fieldOptions().postTags()[0], encoder);
-            mapperHighlighterEntry = new MapperHighlighterEntry(passageFormatter);
-        }
+        Encoder encoder = field.fieldOptions().encoder().equals("html") ? HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
+        CustomPassageFormatter passageFormatter = new CustomPassageFormatter(field.fieldOptions().preTags()[0],
+            field.fieldOptions().postTags()[0], encoder);
 
         List snippets = new ArrayList<>();
         int numberOfFragments;
         try {
-            Analyzer analyzer =
-                context.mapperService().documentMapper(hitContext.hit().getType()).mappers().indexAnalyzer();
+
+            final Analyzer analyzer =
+                getAnalyzer(context.mapperService().documentMapper(hitContext.hit().getType()), fieldMapper.fieldType());
             List fieldValues = HighlightUtils.loadFieldValues(field, fieldMapper, context, hitContext);
-            fieldValues = fieldValues.stream().map(obj -> {
-                if (obj instanceof BytesRef) {
-                    return fieldMapper.fieldType().valueForDisplay(obj).toString();
-                } else {
-                    return obj;
-                }
-            }).collect(Collectors.toList());
+            fieldValues = fieldValues.stream()
+                .map((s) -> convertFieldValue(fieldMapper.fieldType(), s))
+                .collect(Collectors.toList());
             final IndexSearcher searcher = new IndexSearcher(hitContext.reader());
             final CustomUnifiedHighlighter highlighter;
             final String fieldValue = mergeFieldValues(fieldValues, MULTIVAL_SEP_CHAR);
@@ -102,15 +87,14 @@ public class UnifiedHighlighter implements Highlighter {
                 // breaks the text on, so we don't lose the distinction between the different values of a field and we
                 // get back a snippet per value
                 CustomSeparatorBreakIterator breakIterator = new CustomSeparatorBreakIterator(MULTIVAL_SEP_CHAR);
-                highlighter = new CustomUnifiedHighlighter(searcher, analyzer, offsetSource,
-                        mapperHighlighterEntry.passageFormatter, field.fieldOptions().boundaryScannerLocale(),
-                        breakIterator, fieldValue, field.fieldOptions().noMatchSize());
+                highlighter = new CustomUnifiedHighlighter(searcher, analyzer, offsetSource, passageFormatter,
+                    field.fieldOptions().boundaryScannerLocale(), breakIterator, fieldValue, field.fieldOptions().noMatchSize());
                 numberOfFragments = fieldValues.size(); // we are highlighting the whole content, one snippet per value
             } else {
                 //using paragraph separator we make sure that each field value holds a discrete passage for highlighting
                 BreakIterator bi = getBreakIterator(field);
-                highlighter = new CustomUnifiedHighlighter(searcher, analyzer, offsetSource,
-                    mapperHighlighterEntry.passageFormatter, field.fieldOptions().boundaryScannerLocale(), bi,
+                highlighter = new CustomUnifiedHighlighter(searcher, analyzer, offsetSource, passageFormatter,
+                    field.fieldOptions().boundaryScannerLocale(), bi,
                     fieldValue, field.fieldOptions().noMatchSize());
                 numberOfFragments = field.fieldOptions().numberOfFragments();
             }
@@ -210,6 +194,24 @@ public class UnifiedHighlighter implements Highlighter {
         return filteredSnippets;
     }
 
+    static Analyzer getAnalyzer(DocumentMapper docMapper, MappedFieldType type) {
+        if (type instanceof KeywordFieldMapper.KeywordFieldType) {
+            KeywordFieldMapper.KeywordFieldType keywordFieldType = (KeywordFieldMapper.KeywordFieldType) type;
+            if (keywordFieldType.normalizer() != null) {
+                return  keywordFieldType.normalizer();
+            }
+        }
+        return docMapper.mappers().indexAnalyzer();
+    }
+
+    static String convertFieldValue(MappedFieldType type, Object value) {
+        if (value instanceof BytesRef) {
+            return type.valueForDisplay(value).toString();
+        } else {
+            return value.toString();
+        }
+    }
+
     private static String mergeFieldValues(List fieldValues, char valuesSeparator) {
         //postings highlighter accepts all values in a single string, as offsets etc. need to match with content
         //loaded from stored fields, we merge all values using a proper separator
@@ -226,17 +228,4 @@ public class UnifiedHighlighter implements Highlighter {
         }
         return OffsetSource.ANALYSIS;
     }
-
-
-    private static class HighlighterEntry {
-        Map mappers = new HashMap<>();
-    }
-
-    private static class MapperHighlighterEntry {
-        final CustomPassageFormatter passageFormatter;
-
-        private MapperHighlighterEntry(CustomPassageFormatter passageFormatter) {
-            this.passageFormatter = passageFormatter;
-        }
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
index faf1f65f34b..5861e768436 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
@@ -2915,4 +2915,32 @@ public class HighlighterSearchIT extends ESIntegTestCase {
             assertThat(field.getFragments()[0].string(), equalTo("brown"));
         }
     }
+
+    public void testWithNormalizer() throws Exception {
+        Builder builder = Settings.builder()
+            .put(indexSettings())
+            .putList("index.analysis.normalizer.my_normalizer.filter", "lowercase");
+
+        assertAcked(prepareCreate("test").setSettings(builder.build())
+            .addMapping("doc", "keyword",
+                "type=keyword,normalizer=my_normalizer"));
+        ensureGreen();
+
+        client().prepareIndex("test", "doc", "0")
+            .setSource("keyword", "Hello World")
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+
+        for (String highlighterType : new String[] {"unified", "plain"}) {
+            SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(matchQuery("keyword", "hello world"))
+                .highlighter(new HighlightBuilder()
+                    .field(new Field("keyword").highlighterType(highlighterType)))
+                .get();
+            assertHitCount(searchResponse, 1);
+            HighlightField field = searchResponse.getHits().getAt(0).getHighlightFields().get("keyword");
+            assertThat(field.getFragments().length, equalTo(1));
+            assertThat(field.getFragments()[0].string(), equalTo("Hello World"));
+        }
+    }
 }

From e0b1a6544dfb1cce33bbb70980dbbf7e5d11e062 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi 
Date: Mon, 4 Dec 2017 21:32:16 +0100
Subject: [PATCH 166/297] Fix term vectors generator with keyword and
 normalizer (#27608)

This change applies the normalizer defined on the field when building term vectors dynamically on a keyword field.

Fixes #27320
---
 .../index/termvectors/TermVectorsService.java |  7 ++-
 .../action/termvectors/GetTermVectorsIT.java  | 45 +++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index 495f1dc4bdb..91bd994b131 100644
--- a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -217,7 +217,12 @@ public class TermVectorsService  {
         if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) {
             analyzer = mapperService.getIndexAnalyzers().get(perFieldAnalyzer.get(field).toString());
         } else {
-            analyzer = mapperService.fullName(field).indexAnalyzer();
+            MappedFieldType fieldType = mapperService.fullName(field);
+            if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
+                analyzer = ((KeywordFieldMapper.KeywordFieldType) fieldType).normalizer();
+            } else {
+                analyzer = fieldType.indexAnalyzer();
+            }
         }
         if (analyzer == null) {
             analyzer = mapperService.getIndexAnalyzers().getDefaultIndexAnalyzer();
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
index 520c881aa7e..bab9f14fcff 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
@@ -1025,6 +1025,51 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase {
         assertEquals("expected to find term statistics in exactly one shard!", 2, sumDocFreq);
     }
 
+    public void testWithKeywordAndNormalizer() throws IOException, ExecutionException, InterruptedException {
+        // setup indices
+        String[] indexNames = new String[] {"with_tv", "without_tv"};
+        Settings.Builder builder = Settings.builder()
+            .put(indexSettings())
+            .put("index.analysis.analyzer.my_analyzer.tokenizer", "keyword")
+            .putList("index.analysis.analyzer.my_analyzer.filter", "lowercase")
+            .putList("index.analysis.normalizer.my_normalizer.filter", "lowercase");
+        assertAcked(prepareCreate(indexNames[0]).setSettings(builder.build())
+            .addMapping("type1", "field1", "type=text,term_vector=with_positions_offsets,analyzer=my_analyzer"));
+        assertAcked(prepareCreate(indexNames[1]).setSettings(builder.build())
+            .addMapping("type1", "field1", "type=keyword,normalizer=my_normalizer"));
+        ensureGreen();
+
+        // index documents with and without term vectors
+        String[] content = new String[] { "Hello World", "hello world", "HELLO WORLD" };
+
+        List indexBuilders = new ArrayList<>();
+        for (String indexName : indexNames) {
+            for (int id = 0; id < content.length; id++) {
+                indexBuilders.add(client().prepareIndex()
+                    .setIndex(indexName)
+                    .setType("type1")
+                    .setId(String.valueOf(id))
+                    .setSource("field1", content[id]));
+            }
+        }
+        indexRandom(true, indexBuilders);
+
+        // request tvs and compare from each index
+        for (int id = 0; id < content.length; id++) {
+            Fields[] fields = new Fields[2];
+            for (int j = 0; j < indexNames.length; j++) {
+                TermVectorsResponse resp = client().prepareTermVector(indexNames[j], "type1", String.valueOf(id))
+                    .setOffsets(true)
+                    .setPositions(true)
+                    .setSelectedFields("field1")
+                    .get();
+                assertThat("doc with index: " + indexNames[j] + ", type1 and id: " + id, resp.isExists(), equalTo(true));
+                fields[j] = resp.getFields();
+            }
+            compareTermVectors("field1", fields[0], fields[1]);
+        }
+    }
+
     private void checkBestTerms(Terms terms, List expectedTerms) throws IOException {
         final TermsEnum termsEnum = terms.iterator();
         List bestTerms = new ArrayList<>();

From 1ff5ef905554b9f31b762ae8336f3c85f3691d95 Mon Sep 17 00:00:00 2001
From: Lee Hinman 
Date: Mon, 4 Dec 2017 14:14:45 -0700
Subject: [PATCH 167/297] [TEST] Check accounting breaker is equal to segment
 stats rather than 0

If there are existing indices, it may not be 0
---
 .../java/org/elasticsearch/test/ExternalTestCluster.java    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
index 5eb34d96d69..90d58786496 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
@@ -168,8 +168,10 @@ public final class ExternalTestCluster extends TestCluster {
             for (NodeStats stats : nodeStats.getNodes()) {
                 assertThat("Fielddata breaker not reset to 0 on node: " + stats.getNode(),
                         stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
-                assertThat("Accounting breaker not reset to 0 on node: " + stats.getNode(),
-                        stats.getBreaker().getStats(CircuitBreaker.ACCOUNTING).getEstimated(), equalTo(0L));
+                assertThat("Accounting breaker not reset to " + stats.getIndices().getSegments().getMemoryInBytes() +
+                                " on node: " + stats.getNode(),
+                        stats.getBreaker().getStats(CircuitBreaker.ACCOUNTING).getEstimated(),
+                        equalTo(stats.getIndices().getSegments().getMemoryInBytes()));
                 // ExternalTestCluster does not check the request breaker,
                 // because checking it requires a network request, which in
                 // turn increments the breaker, making it non-0

From 4b558636f0e797c17037914201e7970cf00f1625 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen 
Date: Mon, 4 Dec 2017 16:34:34 -0500
Subject: [PATCH 168/297] TEST: Rewrite
 GeoPointParsingTests#testEqualsHashCodeContract (#27634)

The hashCode contract states that equal objects must have equal hash
codes, however the unequal objects are not required to have unequal
hashCodes.

This commit rewrites GeoPointParsingTests#testEqualsHashCodeContract
using#checkEqualsAndHashCode helper.

Closes #27633
---
 .../search/geo/GeoPointParsingTests.java      | 36 ++++++-------------
 1 file changed, 10 insertions(+), 26 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java b/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java
index 6cf3c7efeaf..15fdbe828b0 100644
--- a/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/search/geo/GeoPointParsingTests.java
@@ -29,8 +29,10 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.io.IOException;
+import java.util.function.DoubleSupplier;
 
 import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 import static org.hamcrest.Matchers.is;
 
 public class GeoPointParsingTests  extends ESTestCase {
@@ -57,32 +59,14 @@ public class GeoPointParsingTests  extends ESTestCase {
     }
 
     public void testEqualsHashCodeContract() {
-        // generate a random geopoint
-        final GeoPoint x = RandomGeoGenerator.randomPoint(random());
-        final GeoPoint y = new GeoPoint(x.lat(), x.lon());
-        final GeoPoint z = new GeoPoint(y.lat(), y.lon());
-        // GeoPoint doesn't care about coordinate system bounds, this simply validates inequality
-        final GeoPoint a = new GeoPoint(x.lat() + randomIntBetween(1, 5), x.lon() + randomIntBetween(1, 5));
-
-        /** equality test */
-        // reflexive
-        assertTrue(x.equals(x));
-        // symmetry
-        assertTrue(x.equals(y));
-        // transitivity
-        assertTrue(y.equals(z));
-        assertTrue(x.equals(z));
-        // inequality
-        assertFalse(x.equals(a));
-
-        /** hashCode test */
-        // symmetry
-        assertTrue(x.hashCode() == y.hashCode());
-        // transitivity
-        assertTrue(y.hashCode() == z.hashCode());
-        assertTrue(x.hashCode() == z.hashCode());
-        // inequality
-        assertFalse(x.hashCode() == a.hashCode());
+        // GeoPoint doesn't care about coordinate system bounds, this simply validates equality and hashCode.
+        final DoubleSupplier randomDelta = () -> randomValueOtherThan(0.0, () -> randomDoubleBetween(-1000000, 1000000, true));
+        checkEqualsAndHashCode(RandomGeoGenerator.randomPoint(random()), GeoPoint::new,
+            pt -> new GeoPoint(pt.lat() + randomDelta.getAsDouble(), pt.lon()));
+        checkEqualsAndHashCode(RandomGeoGenerator.randomPoint(random()), GeoPoint::new,
+            pt -> new GeoPoint(pt.lat(), pt.lon() + randomDelta.getAsDouble()));
+        checkEqualsAndHashCode(RandomGeoGenerator.randomPoint(random()), GeoPoint::new,
+            pt -> new GeoPoint(pt.lat() + randomDelta.getAsDouble(), pt.lon() + randomDelta.getAsDouble()));
     }
 
     public void testGeoPointParsing() throws IOException {

From 72800bb90b445f9b9b359c89bdf066be79c88864 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Mon, 4 Dec 2017 16:38:45 -0500
Subject: [PATCH 169/297] Fix Lucene version for 6.2.0 constant in master

This commit fixes the Lucene version constant in master for the 6.2.0
version.

Relates #27658
---
 core/src/main/java/org/elasticsearch/Version.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index 372d88c75cd..a4507fc91e2 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -134,7 +134,7 @@ public class Version implements Comparable {
     public static final int V_6_1_0_ID = 6010099;
     public static final Version V_6_1_0 = new Version(V_6_1_0_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
     public static final int V_6_2_0_ID = 6020099;
-    public static final Version V_6_2_0 = new Version(V_6_2_0_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
+    public static final Version V_6_2_0 = new Version(V_6_2_0_ID, org.apache.lucene.util.Version.LUCENE_7_2_0);
     public static final int V_7_0_0_alpha1_ID = 7000001;
     public static final Version V_7_0_0_alpha1 =
         new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_2_0);

From d3721c48c3f06bb0c44763de4442da773c4e1839 Mon Sep 17 00:00:00 2001
From: Lee Hinman 
Date: Mon, 4 Dec 2017 15:42:43 -0700
Subject: [PATCH 170/297] [TEST] Add AwaitsFix for
 FullClusterRestartIT.testRecovery

See: #27649
---
 .../java/org/elasticsearch/upgrades/FullClusterRestartIT.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
index 00d700d2d52..568ae43c323 100644
--- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -615,6 +615,7 @@ public class FullClusterRestartIT extends ESRestTestCase {
      * Tests recovery of an index with or without a translog and the
      * statistics we gather about that.
      */
+    @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/27649")
     public void testRecovery() throws IOException {
         int count;
         boolean shouldHaveTranslog;

From 963ed25cf599a78f12433ab9520e3172838405b5 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Mon, 4 Dec 2017 18:10:04 -0500
Subject: [PATCH 171/297] Only fsync global checkpoint if needed

In the global checkpoint sync action, we fsync the translog. However,
the last synced global checkpoint might already be equal to the current
global checkpoint in which case the fsyncing the translog is unnecessary
as either the sync needed guard in the translog will skip the translog,
or the translog needs an fsync for another reason that will be picked up
elsewhere (e.g., at the end of a bulk request).

Relates #27652
---
 .../index/seqno/GlobalCheckpointSyncAction.java  | 16 +++++++++++-----
 .../seqno/GlobalCheckpointSyncActionTests.java   | 15 ++++++++++++++-
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java b/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
index 04caf187db9..95e3505e746 100644
--- a/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
+++ b/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
@@ -44,6 +44,8 @@ import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 /**
  * Background global checkpoint sync action initiated when a shard goes inactive. This is needed because while we send the global checkpoint
  * on every replication operation, after the last operation completes the global checkpoint could advance but without a follow-up operation
@@ -117,18 +119,22 @@ public class GlobalCheckpointSyncAction extends TransportReplicationAction<
     @Override
     protected PrimaryResult shardOperationOnPrimary(
             final Request request, final IndexShard indexShard) throws Exception {
-        if (indexShard.getTranslogDurability() == Translog.Durability.REQUEST) {
-            indexShard.getTranslog().sync();
-        }
+        maybeSyncTranslog(indexShard);
         return new PrimaryResult<>(request, new ReplicationResponse());
     }
 
     @Override
     protected ReplicaResult shardOperationOnReplica(final Request request, final IndexShard indexShard) throws Exception {
-        if (indexShard.getTranslogDurability() == Translog.Durability.REQUEST) {
+        maybeSyncTranslog(indexShard);
+        return new ReplicaResult();
+    }
+
+    private void maybeSyncTranslog(final IndexShard indexShard) throws IOException {
+        final Translog translog = indexShard.getTranslog();
+        if (indexShard.getTranslogDurability() == Translog.Durability.REQUEST &&
+                translog.getLastSyncedGlobalCheckpoint() < indexShard.getGlobalCheckpoint()) {
             indexShard.getTranslog().sync();
         }
-        return new ReplicaResult();
     }
 
     public static final class Request extends ReplicationRequest {
diff --git a/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java b/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
index c327e47c30c..71faecfcea5 100644
--- a/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
+++ b/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
@@ -93,6 +93,19 @@ public class GlobalCheckpointSyncActionTests extends ESTestCase {
         final Translog translog = mock(Translog.class);
         when(indexShard.getTranslog()).thenReturn(translog);
 
+        final long globalCheckpoint = randomIntBetween(Math.toIntExact(SequenceNumbers.NO_OPS_PERFORMED), Integer.MAX_VALUE);
+        final long lastSyncedGlobalCheckpoint;
+        if (randomBoolean() && globalCheckpoint != SequenceNumbers.NO_OPS_PERFORMED) {
+            lastSyncedGlobalCheckpoint =
+                    randomIntBetween(Math.toIntExact(SequenceNumbers.NO_OPS_PERFORMED), Math.toIntExact(globalCheckpoint) - 1);
+            assert lastSyncedGlobalCheckpoint < globalCheckpoint;
+        } else {
+            lastSyncedGlobalCheckpoint = globalCheckpoint;
+        }
+
+        when(indexShard.getGlobalCheckpoint()).thenReturn(globalCheckpoint);
+        when(translog.getLastSyncedGlobalCheckpoint()).thenReturn(lastSyncedGlobalCheckpoint);
+
         final GlobalCheckpointSyncAction action = new GlobalCheckpointSyncAction(
             Settings.EMPTY,
             transportService,
@@ -109,7 +122,7 @@ public class GlobalCheckpointSyncActionTests extends ESTestCase {
             action.shardOperationOnReplica(new GlobalCheckpointSyncAction.Request(indexShard.shardId()), indexShard);
         }
 
-        if (durability == Translog.Durability.ASYNC) {
+        if (durability == Translog.Durability.ASYNC || lastSyncedGlobalCheckpoint == globalCheckpoint) {
             verify(translog, never()).sync();
         } else {
             verify(translog).sync();

From 8635f68ece81db9ca6c51e45aba0ab13b93daecc Mon Sep 17 00:00:00 2001
From: Jim Ferenczi 
Date: Tue, 5 Dec 2017 01:32:53 +0100
Subject: [PATCH 172/297] Fix term vectors generator with keyword and
 normalizer (bis)

Fallback on the index analyzer if the normalizer is null.

Closes #27320
---
 .../index/termvectors/TermVectorsService.java          |  3 ++-
 .../action/termvectors/GetTermVectorsIT.java           | 10 ++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index 91bd994b131..d527fa83501 100644
--- a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -219,7 +219,8 @@ public class TermVectorsService  {
         } else {
             MappedFieldType fieldType = mapperService.fullName(field);
             if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
-                analyzer = ((KeywordFieldMapper.KeywordFieldType) fieldType).normalizer();
+                KeywordFieldMapper.KeywordFieldType keywordFieldType = (KeywordFieldMapper.KeywordFieldType) fieldType;
+                analyzer = keywordFieldType.normalizer() == null ? keywordFieldType.indexAnalyzer() : keywordFieldType.normalizer();
             } else {
                 analyzer = fieldType.indexAnalyzer();
             }
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
index bab9f14fcff..79cc13594e9 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
@@ -1034,9 +1034,10 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase {
             .putList("index.analysis.analyzer.my_analyzer.filter", "lowercase")
             .putList("index.analysis.normalizer.my_normalizer.filter", "lowercase");
         assertAcked(prepareCreate(indexNames[0]).setSettings(builder.build())
-            .addMapping("type1", "field1", "type=text,term_vector=with_positions_offsets,analyzer=my_analyzer"));
+            .addMapping("type1", "field1", "type=text,term_vector=with_positions_offsets,analyzer=my_analyzer",
+                "field2", "type=text,term_vector=with_positions_offsets,analyzer=keyword"));
         assertAcked(prepareCreate(indexNames[1]).setSettings(builder.build())
-            .addMapping("type1", "field1", "type=keyword,normalizer=my_normalizer"));
+            .addMapping("type1", "field1", "type=keyword,normalizer=my_normalizer", "field2", "type=keyword"));
         ensureGreen();
 
         // index documents with and without term vectors
@@ -1049,7 +1050,7 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase {
                     .setIndex(indexName)
                     .setType("type1")
                     .setId(String.valueOf(id))
-                    .setSource("field1", content[id]));
+                    .setSource("field1", content[id], "field2", content[id]));
             }
         }
         indexRandom(true, indexBuilders);
@@ -1061,12 +1062,13 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase {
                 TermVectorsResponse resp = client().prepareTermVector(indexNames[j], "type1", String.valueOf(id))
                     .setOffsets(true)
                     .setPositions(true)
-                    .setSelectedFields("field1")
+                    .setSelectedFields("field1", "field2")
                     .get();
                 assertThat("doc with index: " + indexNames[j] + ", type1 and id: " + id, resp.isExists(), equalTo(true));
                 fields[j] = resp.getFields();
             }
             compareTermVectors("field1", fields[0], fields[1]);
+            compareTermVectors("field2", fields[0], fields[1]);
         }
     }
 

From 2208a1a7b5dd249e85be515b80f62be35f158008 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Mon, 4 Dec 2017 19:53:14 -0500
Subject: [PATCH 173/297] Detect mktemp from coreutils

GNU mktemp and BSD mktemp have different command line flags. On some
macOS systems users have mktemp from coreutils in their PATH overriding
the system mktemp from BSD. This commit adds detection for the coreutils
mktemp versus the BSD mktemp and uses the appropriate syntax based on
the detection.

Relates #27659
---
 distribution/src/main/resources/bin/elasticsearch-env | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch-env b/distribution/src/main/resources/bin/elasticsearch-env
index 2b376bd47b3..487dcf20922 100644
--- a/distribution/src/main/resources/bin/elasticsearch-env
+++ b/distribution/src/main/resources/bin/elasticsearch-env
@@ -75,9 +75,13 @@ if [ -z "$ES_PATH_CONF" ]; then
 fi
 
 if [ -z "$ES_TMPDIR" ]; then
-  if [ "`uname`" == "Darwin" ]; then
-    ES_TMPDIR=`mktemp -d -t elasticsearch`
+  set +e
+  mktemp --version 2>&1 | grep coreutils > /dev/null
+  mktemp_coreutils=$?
+  set -e
+  if [ $mktemp_coreutils -eq 0 ]; then
+    ES_TMPDIR=`mktemp -d --tmpdir "elasticearch.XXXXXXXX"`
   else
-    ES_TMPDIR=`mktemp -d -t elasticsearch.XXXXXXXX`
+    ES_TMPDIR=`mktemp -d -t elasticsearch`
   fi
 fi

From 99db3913440cb49ebd568bbb8caf87bf43cb0615 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Mon, 4 Dec 2017 21:21:57 -0500
Subject: [PATCH 174/297] Add explicit coreutils dependency

The RPM and Debian packages depend on coreutils (for mktemp among
others). This commit adds an explicit package dependency on coreutils.

Relates #27660
---
 distribution/build.gradle | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distribution/build.gradle b/distribution/build.gradle
index b7fa48561da..a9298718a39 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -392,6 +392,7 @@ configure(distributions.findAll { ['deb', 'rpm'].contains(it.name) }) {
     } else if (project.name == 'deb') {
       requires('bash')
     }
+    requires('coreutils')
 
     into '/usr/share/elasticsearch'
     fileMode 0644

From c203cff6924edffc4345412072597d1ff7da52c7 Mon Sep 17 00:00:00 2001
From: Boaz Leskes 
Date: Fri, 1 Dec 2017 15:12:51 +0100
Subject: [PATCH 175/297] fix java9 compilation

---
 .../indices/recovery/RecoverySourceHandlerTests.java        | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
index c04d69bbed2..cf5f24d2a6e 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
@@ -211,8 +211,10 @@ public class RecoverySourceHandlerTests extends ESTestCase {
         assertThat(result.totalOperations, equalTo(expectedOps));
         final ArgumentCaptor shippedOpsCaptor = ArgumentCaptor.forClass(List.class);
         verify(recoveryTarget).indexTranslogOperations(shippedOpsCaptor.capture(), ArgumentCaptor.forClass(Integer.class).capture());
-        List shippedOps = shippedOpsCaptor.getAllValues().stream()
-            .flatMap(List::stream).map(o -> (Translog.Operation) o).collect(Collectors.toList());
+        List shippedOps = new ArrayList<>();
+        for (List list: shippedOpsCaptor.getAllValues()) {
+            shippedOps.addAll(list);
+        }
         shippedOps.sort(Comparator.comparing(Translog.Operation::seqNo));
         assertThat(shippedOps.size(), equalTo(expectedOps));
         for (int i = 0; i < shippedOps.size(); i++) {

From 4d78e1a9ad4b84d14fa7c7a859d4334d84768c63 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen 
Date: Mon, 6 Nov 2017 11:26:04 +0100
Subject: [PATCH 176/297] Added msearch api to high level client

---
 .../org/elasticsearch/client/Request.java     |  14 ++
 .../client/RestHighLevelClient.java           |  24 ++
 .../elasticsearch/client/RequestTests.java    |  63 ++++-
 .../org/elasticsearch/client/SearchIT.java    | 218 +++++++++++++++++-
 .../action/search/MultiSearchRequest.java     | 183 +++++++++++++++
 .../action/search/MultiSearchResponse.java    |  55 +++++
 .../action/search/SearchResponse.java         |  26 ++-
 .../search/TransportMultiSearchAction.java    |   2 +-
 .../action/search/RestMultiSearchAction.java  | 102 +-------
 .../search/MultiSearchRequestTests.java       |  95 ++++++++
 .../search/MultiSearchResponseTests.java      |  87 +++++++
 .../RestMultiSearchTemplateAction.java        |  16 +-
 12 files changed, 759 insertions(+), 126 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
index e2a6dcac20b..05ce54437a4 100755
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
@@ -35,6 +35,7 @@ import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
@@ -49,6 +50,7 @@ import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -381,6 +383,18 @@ public final class Request {
         return new Request("DELETE", "/_search/scroll", Collections.emptyMap(), entity);
     }
 
+    static Request multiSearch(MultiSearchRequest multiSearchRequest) throws IOException {
+        Params params = Params.builder();
+        params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true");
+        if (multiSearchRequest.maxConcurrentSearchRequests() != MultiSearchRequest.MAX_CONCURRENT_SEARCH_REQUESTS_DEFAULT) {
+            params.putParam("max_concurrent_searches", Integer.toString(multiSearchRequest.maxConcurrentSearchRequests()));
+        }
+        XContent xContent = REQUEST_BODY_CONTENT_TYPE.xContent();
+        byte[] source = MultiSearchRequest.writeMultiLineFormat(multiSearchRequest, xContent);
+        HttpEntity entity = new ByteArrayEntity(source, createContentType(xContent.type()));
+        return new Request("GET", "/_msearch", params.getParams(), entity);
+    }
+
     private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
         return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
index 2ebaf2cf342..29ab7f90ff5 100755
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -38,6 +38,8 @@ import org.elasticsearch.action.main.MainRequest;
 import org.elasticsearch.action.main.MainResponse;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.MultiSearchRequest;
+import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
@@ -377,6 +379,28 @@ public class RestHighLevelClient implements Closeable {
         performRequestAsyncAndParseEntity(searchRequest, Request::search, SearchResponse::fromXContent, listener, emptySet(), headers);
     }
 
+    /**
+     * Executes a multi search using the msearch API
+     *
+     * See Multi search API on
+     * elastic.co
+     */
+    public final MultiSearchResponse multiSearch(MultiSearchRequest multiSearchRequest, Header... headers) throws IOException {
+        return performRequestAndParseEntity(multiSearchRequest, Request::multiSearch, MultiSearchResponse::fromXContext,
+                emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously executes a multi search using the msearch API
+     *
+     * See Multi search API on
+     * elastic.co
+     */
+    public final void multiSearchAsync(MultiSearchRequest searchRequest, ActionListener listener, Header... headers) {
+        performRequestAsyncAndParseEntity(searchRequest, Request::multiSearch, MultiSearchResponse::fromXContext, listener,
+                emptySet(), headers);
+    }
+
     /**
      * Executes a search using the Search Scroll API
      *
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
index 3be250d513d..f72a7cb4dbf 100755
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.SearchType;
@@ -42,6 +43,7 @@ import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
 import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -56,6 +58,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -72,16 +75,21 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Modifier;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.StringJoiner;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
 
 import static java.util.Collections.singletonMap;
+import static org.elasticsearch.client.Request.REQUEST_BODY_CONTENT_TYPE;
 import static org.elasticsearch.client.Request.enforceSameContentType;
+import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 
 public class RequestTests extends ESTestCase {
@@ -771,6 +779,55 @@ public class RequestTests extends ESTestCase {
         }
     }
 
+    public void testMultiSearch() throws IOException {
+        int numberOfSearchRequests = randomIntBetween(0, 32);
+        MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
+        for (int i = 0; i < numberOfSearchRequests; i++) {
+            SearchRequest searchRequest = randomSearchRequest(() -> {
+                // No need to return a very complex SearchSourceBuilder here, that is tested elsewhere
+                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+                searchSourceBuilder.from(randomInt(10));
+                searchSourceBuilder.size(randomIntBetween(20, 100));
+                return searchSourceBuilder;
+            });
+            // scroll is not supported in the current msearch api, so unset it:
+            searchRequest.scroll((Scroll) null);
+            // only expand_wildcards, ignore_unavailable and allow_no_indices can be specified from msearch api, so unset other options:
+            IndicesOptions randomlyGenerated = searchRequest.indicesOptions();
+            IndicesOptions msearchDefault = new MultiSearchRequest().indicesOptions();
+            searchRequest.indicesOptions(IndicesOptions.fromOptions(
+                    randomlyGenerated.ignoreUnavailable(), randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(),
+                    randomlyGenerated.expandWildcardsClosed(), msearchDefault.allowAliasesToMultipleIndices(),
+                    msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases()
+            ));
+            multiSearchRequest.add(searchRequest);
+        }
+
+        Map expectedParams = new HashMap<>();
+        expectedParams.put(RestSearchAction.TYPED_KEYS_PARAM, "true");
+        if (randomBoolean()) {
+            multiSearchRequest.maxConcurrentSearchRequests(randomIntBetween(1, 8));
+            expectedParams.put("max_concurrent_searches", Integer.toString(multiSearchRequest.maxConcurrentSearchRequests()));
+        }
+
+        Request request = Request.multiSearch(multiSearchRequest);
+        assertEquals("/_msearch", request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+
+        List requests = new ArrayList<>();
+        CheckedBiConsumer consumer = (searchRequest, p) -> {
+            SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(p);
+            if (searchSourceBuilder.equals(new SearchSourceBuilder()) == false) {
+                searchRequest.source(searchSourceBuilder);
+            }
+            requests.add(searchRequest);
+        };
+        MultiSearchRequest.readMultiLineFormat(new BytesArray(EntityUtils.toByteArray(request.getEntity())),
+                REQUEST_BODY_CONTENT_TYPE.xContent(), consumer, null, multiSearchRequest.indicesOptions(), null, null,
+                null, xContentRegistry(), true);
+        assertEquals(requests, multiSearchRequest.requests());
+    }
+
     public void testSearchScroll() throws IOException {
         SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
         searchScrollRequest.scrollId(randomAlphaOfLengthBetween(5, 10));
@@ -782,7 +839,7 @@ public class RequestTests extends ESTestCase {
         assertEquals("/_search/scroll", request.getEndpoint());
         assertEquals(0, request.getParameters().size());
         assertToXContentBody(searchScrollRequest, request.getEntity());
-        assertEquals(Request.REQUEST_BODY_CONTENT_TYPE.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
+        assertEquals(REQUEST_BODY_CONTENT_TYPE.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
     }
 
     public void testClearScroll() throws IOException {
@@ -796,11 +853,11 @@ public class RequestTests extends ESTestCase {
         assertEquals("/_search/scroll", request.getEndpoint());
         assertEquals(0, request.getParameters().size());
         assertToXContentBody(clearScrollRequest, request.getEntity());
-        assertEquals(Request.REQUEST_BODY_CONTENT_TYPE.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
+        assertEquals(REQUEST_BODY_CONTENT_TYPE.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
     }
 
     private static void assertToXContentBody(ToXContent expectedBody, HttpEntity actualEntity) throws IOException {
-        BytesReference expectedBytes = XContentHelper.toXContent(expectedBody, Request.REQUEST_BODY_CONTENT_TYPE, false);
+        BytesReference expectedBytes = XContentHelper.toXContent(expectedBody, REQUEST_BODY_CONTENT_TYPE, false);
         assertEquals(XContentType.JSON.mediaTypeWithoutParameters(), actualEntity.getContentType().getValue());
         assertEquals(expectedBytes, new BytesArray(EntityUtils.toByteArray(actualEntity)));
     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
index 289ebf372d8..3e72c7c64b6 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
@@ -23,20 +23,30 @@ import org.apache.http.HttpEntity;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.nio.entity.NStringEntity;
+import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.MultiSearchRequest;
+import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
+import org.elasticsearch.index.query.NestedQueryBuilder;
+import org.elasticsearch.index.query.ScriptQueryBuilder;
+import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.join.aggregations.Children;
 import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
@@ -45,10 +55,12 @@ import org.elasticsearch.search.aggregations.matrix.stats.MatrixStats;
 import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.hamcrest.Matchers;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -64,6 +76,7 @@ import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.nullValue;
 
 public class SearchIT extends ESRestHighLevelClientTestCase {
 
@@ -80,10 +93,24 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
         StringEntity doc5 = new StringEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}", ContentType.APPLICATION_JSON);
         client().performRequest("PUT", "/index/type/5", Collections.emptyMap(), doc5);
         client().performRequest("POST", "/index/_refresh");
+
+        StringEntity doc = new StringEntity("{\"field\":\"value1\"}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index1/doc/1", Collections.emptyMap(), doc);
+        doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index1/doc/2", Collections.emptyMap(), doc);
+        doc = new StringEntity("{\"field\":\"value1\"}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index2/doc/3", Collections.emptyMap(), doc);
+        doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index2/doc/4", Collections.emptyMap(), doc);
+        doc = new StringEntity("{\"field\":\"value1\"}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index3/doc/5", Collections.emptyMap(), doc);
+        doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index3/doc/6", Collections.emptyMap(), doc);
+        client().performRequest("POST", "/index1,index2,index3/_refresh");
     }
 
     public void testSearchNoQuery() throws IOException {
-        SearchRequest searchRequest = new SearchRequest();
+        SearchRequest searchRequest = new SearchRequest("index");
         SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
         assertSearchHeader(searchResponse);
         assertNull(searchResponse.getAggregations());
@@ -106,7 +133,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
     }
 
     public void testSearchMatchQuery() throws IOException {
-        SearchRequest searchRequest = new SearchRequest();
+        SearchRequest searchRequest = new SearchRequest("index");
         searchRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("num", 10)));
         SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
         assertSearchHeader(searchResponse);
@@ -164,7 +191,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
             assertEquals(RestStatus.BAD_REQUEST, exception.status());
         }
 
-        SearchRequest searchRequest = new SearchRequest();
+        SearchRequest searchRequest = new SearchRequest("index");
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
         searchSourceBuilder.aggregation(new RangeAggregationBuilder("agg1").field("num")
                 .addRange("first", 0, 30).addRange("second", 31, 200));
@@ -193,7 +220,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
     }
 
     public void testSearchWithTermsAndRangeAgg() throws IOException {
-        SearchRequest searchRequest = new SearchRequest();
+        SearchRequest searchRequest = new SearchRequest("index");
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
         TermsAggregationBuilder agg = new TermsAggregationBuilder("agg1", ValueType.STRING).field("type.keyword");
         agg.subAggregation(new RangeAggregationBuilder("subagg").field("num")
@@ -247,7 +274,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
     }
 
     public void testSearchWithMatrixStats() throws IOException {
-        SearchRequest searchRequest = new SearchRequest();
+        SearchRequest searchRequest = new SearchRequest("index");
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
         searchSourceBuilder.aggregation(new MatrixStatsAggregationBuilder("agg1").fields(Arrays.asList("num", "num2")));
         searchSourceBuilder.size(0);
@@ -374,7 +401,7 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
     }
 
     public void testSearchWithSuggest() throws IOException {
-        SearchRequest searchRequest = new SearchRequest();
+        SearchRequest searchRequest = new SearchRequest("index");
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
         searchSourceBuilder.suggest(new SuggestBuilder().addSuggestion("sugg1", new PhraseSuggestionBuilder("type"))
                 .setGlobalText("type"));
@@ -464,6 +491,185 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
         }
     }
 
+    public void testMultiSearch() throws Exception {
+        MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
+        SearchRequest searchRequest1 = new SearchRequest("index1");
+        searchRequest1.source().sort("_id", SortOrder.ASC);
+        multiSearchRequest.add(searchRequest1);
+        SearchRequest searchRequest2 = new SearchRequest("index2");
+        searchRequest2.source().sort("_id", SortOrder.ASC);
+        multiSearchRequest.add(searchRequest2);
+        SearchRequest searchRequest3 = new SearchRequest("index3");
+        searchRequest3.source().sort("_id", SortOrder.ASC);
+        multiSearchRequest.add(searchRequest3);
+
+        MultiSearchResponse multiSearchResponse =
+                execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync);
+        assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
+        assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3));
+
+        assertThat(multiSearchResponse.getResponses()[0].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[0].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[0].getResponse());
+        assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getTotalHits(), Matchers.equalTo(2L));
+        assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getAt(0).getId(), Matchers.equalTo("1"));
+        assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getAt(1).getId(), Matchers.equalTo("2"));
+
+        assertThat(multiSearchResponse.getResponses()[1].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[1].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[1].getResponse());
+        assertThat(multiSearchResponse.getResponses()[1].getResponse().getHits().getTotalHits(), Matchers.equalTo(2L));
+        assertThat(multiSearchResponse.getResponses()[1].getResponse().getHits().getAt(0).getId(), Matchers.equalTo("3"));
+        assertThat(multiSearchResponse.getResponses()[1].getResponse().getHits().getAt(1).getId(), Matchers.equalTo("4"));
+
+        assertThat(multiSearchResponse.getResponses()[2].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[2].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[2].getResponse());
+        assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getTotalHits(), Matchers.equalTo(2L));
+        assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getAt(0).getId(), Matchers.equalTo("5"));
+        assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getAt(1).getId(), Matchers.equalTo("6"));
+    }
+
+    public void testMultiSearch_withAgg() throws Exception {
+        MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
+        SearchRequest searchRequest1 = new SearchRequest("index1");
+        searchRequest1.source().size(0).aggregation(new TermsAggregationBuilder("name", ValueType.STRING).field("field.keyword")
+                .order(BucketOrder.key(true)));
+        multiSearchRequest.add(searchRequest1);
+        SearchRequest searchRequest2 = new SearchRequest("index2");
+        searchRequest2.source().size(0).aggregation(new TermsAggregationBuilder("name", ValueType.STRING).field("field.keyword")
+                .order(BucketOrder.key(true)));
+        multiSearchRequest.add(searchRequest2);
+        SearchRequest searchRequest3 = new SearchRequest("index3");
+        searchRequest3.source().size(0).aggregation(new TermsAggregationBuilder("name", ValueType.STRING).field("field.keyword")
+                .order(BucketOrder.key(true)));
+        multiSearchRequest.add(searchRequest3);
+
+        MultiSearchResponse multiSearchResponse =
+                execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync);
+        assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
+        assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3));
+
+        assertThat(multiSearchResponse.getResponses()[0].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[0].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[0].getResponse());
+        assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getTotalHits(), Matchers.equalTo(2L));
+        assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getHits().length, Matchers.equalTo(0));
+        Terms terms = multiSearchResponse.getResponses()[0].getResponse().getAggregations().get("name");
+        assertThat(terms.getBuckets().size(), Matchers.equalTo(2));
+        assertThat(terms.getBuckets().get(0).getKeyAsString(), Matchers.equalTo("value1"));
+        assertThat(terms.getBuckets().get(1).getKeyAsString(), Matchers.equalTo("value2"));
+
+        assertThat(multiSearchResponse.getResponses()[1].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[1].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[0].getResponse());
+        assertThat(multiSearchResponse.getResponses()[1].getResponse().getHits().getTotalHits(), Matchers.equalTo(2L));
+        assertThat(multiSearchResponse.getResponses()[1].getResponse().getHits().getHits().length, Matchers.equalTo(0));
+        terms = multiSearchResponse.getResponses()[1].getResponse().getAggregations().get("name");
+        assertThat(terms.getBuckets().size(), Matchers.equalTo(2));
+        assertThat(terms.getBuckets().get(0).getKeyAsString(), Matchers.equalTo("value1"));
+        assertThat(terms.getBuckets().get(1).getKeyAsString(), Matchers.equalTo("value2"));
+
+        assertThat(multiSearchResponse.getResponses()[2].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[2].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[0].getResponse());
+        assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getTotalHits(), Matchers.equalTo(2L));
+        assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getHits().length, Matchers.equalTo(0));
+        terms = multiSearchResponse.getResponses()[2].getResponse().getAggregations().get("name");
+        assertThat(terms.getBuckets().size(), Matchers.equalTo(2));
+        assertThat(terms.getBuckets().get(0).getKeyAsString(), Matchers.equalTo("value1"));
+        assertThat(terms.getBuckets().get(1).getKeyAsString(), Matchers.equalTo("value2"));
+    }
+
+    public void testMultiSearch_withQuery() throws Exception {
+        MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
+        SearchRequest searchRequest1 = new SearchRequest("index1");
+        searchRequest1.source().query(new TermsQueryBuilder("field", "value2"));
+        multiSearchRequest.add(searchRequest1);
+        SearchRequest searchRequest2 = new SearchRequest("index2");
+        searchRequest2.source().query(new TermsQueryBuilder("field", "value2"));
+        multiSearchRequest.add(searchRequest2);
+        SearchRequest searchRequest3 = new SearchRequest("index3");
+        searchRequest3.source().query(new TermsQueryBuilder("field", "value2"));
+        multiSearchRequest.add(searchRequest3);
+
+        MultiSearchResponse multiSearchResponse =
+                execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync);
+        assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
+        assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3));
+
+        assertThat(multiSearchResponse.getResponses()[0].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[0].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[0].getResponse());
+        assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getTotalHits(), Matchers.equalTo(1L));
+        assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getAt(0).getId(), Matchers.equalTo("2"));
+
+        assertThat(multiSearchResponse.getResponses()[1].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[1].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[1].getResponse());
+        assertThat(multiSearchResponse.getResponses()[1].getResponse().getHits().getTotalHits(), Matchers.equalTo(1L));
+        assertThat(multiSearchResponse.getResponses()[1].getResponse().getHits().getAt(0).getId(), Matchers.equalTo("4"));
+
+        assertThat(multiSearchResponse.getResponses()[2].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[2].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[2].getResponse());
+        assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getTotalHits(), Matchers.equalTo(1L));
+        assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getAt(0).getId(), Matchers.equalTo("6"));
+
+        searchRequest1.source().highlighter(new HighlightBuilder().field("field"));
+        searchRequest2.source().highlighter(new HighlightBuilder().field("field"));
+        searchRequest3.source().highlighter(new HighlightBuilder().field("field"));
+        multiSearchResponse = execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync);
+        assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
+        assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3));
+
+        assertThat(multiSearchResponse.getResponses()[0].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[0].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[0].getResponse());
+        assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getTotalHits(), Matchers.equalTo(1L));
+        assertThat(multiSearchResponse.getResponses()[0].getResponse().getHits().getAt(0).getHighlightFields()
+                .get("field").fragments()[0].string(), Matchers.equalTo("value2"));
+
+        assertThat(multiSearchResponse.getResponses()[1].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[1].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[1].getResponse());
+        assertThat(multiSearchResponse.getResponses()[1].getResponse().getHits().getTotalHits(), Matchers.equalTo(1L));
+        assertThat(multiSearchResponse.getResponses()[1].getResponse().getHits().getAt(0).getId(), Matchers.equalTo("4"));
+        assertThat(multiSearchResponse.getResponses()[1].getResponse().getHits().getAt(0).getHighlightFields()
+                .get("field").fragments()[0].string(), Matchers.equalTo("value2"));
+
+        assertThat(multiSearchResponse.getResponses()[2].getFailure(), Matchers.nullValue());
+        assertThat(multiSearchResponse.getResponses()[2].isFailure(), Matchers.is(false));
+        SearchIT.assertSearchHeader(multiSearchResponse.getResponses()[2].getResponse());
+        assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getTotalHits(), Matchers.equalTo(1L));
+        assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getAt(0).getId(), Matchers.equalTo("6"));
+        assertThat(multiSearchResponse.getResponses()[2].getResponse().getHits().getAt(0).getHighlightFields()
+                .get("field").fragments()[0].string(), Matchers.equalTo("value2"));
+    }
+
+    public void testMultiSearch_failure() throws Exception {
+        MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
+        SearchRequest searchRequest1 = new SearchRequest("index1");
+        searchRequest1.source().query(new ScriptQueryBuilder(new Script(ScriptType.INLINE, "invalid", "code", Collections.emptyMap())));
+        multiSearchRequest.add(searchRequest1);
+        SearchRequest searchRequest2 = new SearchRequest("index2");
+        searchRequest2.source().query(new ScriptQueryBuilder(new Script(ScriptType.INLINE, "invalid", "code", Collections.emptyMap())));
+        multiSearchRequest.add(searchRequest2);
+
+        MultiSearchResponse multiSearchResponse =
+                execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync);
+        assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L));
+        assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(2));
+
+        assertThat(multiSearchResponse.getResponses()[0].isFailure(), Matchers.is(true));
+        assertThat(multiSearchResponse.getResponses()[0].getFailure().getMessage(), containsString("search_phase_execution_exception"));
+        assertThat(multiSearchResponse.getResponses()[0].getResponse(), nullValue());
+
+        assertThat(multiSearchResponse.getResponses()[1].isFailure(), Matchers.is(true));
+        assertThat(multiSearchResponse.getResponses()[1].getFailure().getMessage(), containsString("search_phase_execution_exception"));
+        assertThat(multiSearchResponse.getResponses()[1].getResponse(), nullValue());
+    }
+
     private static void assertSearchHeader(SearchResponse searchResponse) {
         assertThat(searchResponse.getTook().nanos(), greaterThanOrEqualTo(0L));
         assertEquals(0, searchResponse.getFailedShards());
diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
index 76f73bde4b6..7772b245658 100644
--- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
@@ -23,20 +23,36 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue;
 
 /**
  * A multi search API request.
  */
 public class MultiSearchRequest extends ActionRequest implements CompositeIndicesRequest {
 
+    public static final int MAX_CONCURRENT_SEARCH_REQUESTS_DEFAULT = 0;
+
     private int maxConcurrentSearchRequests = 0;
     private List requests = new ArrayList<>();
 
@@ -131,4 +147,171 @@ public class MultiSearchRequest extends ActionRequest implements CompositeIndice
             request.writeTo(out);
         }
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        MultiSearchRequest that = (MultiSearchRequest) o;
+        return maxConcurrentSearchRequests == that.maxConcurrentSearchRequests &&
+                Objects.equals(requests, that.requests) &&
+                Objects.equals(indicesOptions, that.indicesOptions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(maxConcurrentSearchRequests, requests, indicesOptions);
+    }
+
+    public static void readMultiLineFormat(BytesReference data,
+                                           XContent xContent,
+                                           CheckedBiConsumer consumer,
+                                           String[] indices,
+                                           IndicesOptions indicesOptions,
+                                           String[] types,
+                                           String routing,
+                                           String searchType,
+                                           NamedXContentRegistry registry,
+                                           boolean allowExplicitIndex) throws IOException {
+        int from = 0;
+        int length = data.length();
+        byte marker = xContent.streamSeparator();
+        while (true) {
+            int nextMarker = findNextMarker(marker, from, data, length);
+            if (nextMarker == -1) {
+                break;
+            }
+            // support first line with \n
+            if (nextMarker == 0) {
+                from = nextMarker + 1;
+                continue;
+            }
+
+            SearchRequest searchRequest = new SearchRequest();
+            if (indices != null) {
+                searchRequest.indices(indices);
+            }
+            if (indicesOptions != null) {
+                searchRequest.indicesOptions(indicesOptions);
+            }
+            if (types != null && types.length > 0) {
+                searchRequest.types(types);
+            }
+            if (routing != null) {
+                searchRequest.routing(routing);
+            }
+            if (searchType != null) {
+                searchRequest.searchType(searchType);
+            }
+            IndicesOptions defaultOptions = SearchRequest.DEFAULT_INDICES_OPTIONS;
+            // now parse the action
+            if (nextMarker - from > 0) {
+                try (XContentParser parser = xContent.createParser(registry, data.slice(from, nextMarker - from))) {
+                    Map source = parser.map();
+                    for (Map.Entry entry : source.entrySet()) {
+                        Object value = entry.getValue();
+                        if ("index".equals(entry.getKey()) || "indices".equals(entry.getKey())) {
+                            if (!allowExplicitIndex) {
+                                throw new IllegalArgumentException("explicit index in multi search is not allowed");
+                            }
+                            searchRequest.indices(nodeStringArrayValue(value));
+                        } else if ("type".equals(entry.getKey()) || "types".equals(entry.getKey())) {
+                            searchRequest.types(nodeStringArrayValue(value));
+                        } else if ("search_type".equals(entry.getKey()) || "searchType".equals(entry.getKey())) {
+                            searchRequest.searchType(nodeStringValue(value, null));
+                        } else if ("request_cache".equals(entry.getKey()) || "requestCache".equals(entry.getKey())) {
+                            searchRequest.requestCache(nodeBooleanValue(value, entry.getKey()));
+                        } else if ("preference".equals(entry.getKey())) {
+                            searchRequest.preference(nodeStringValue(value, null));
+                        } else if ("routing".equals(entry.getKey())) {
+                            searchRequest.routing(nodeStringValue(value, null));
+                        }
+                    }
+                    defaultOptions = IndicesOptions.fromMap(source, defaultOptions);
+                }
+            }
+            searchRequest.indicesOptions(defaultOptions);
+
+            // move pointers
+            from = nextMarker + 1;
+            // now for the body
+            nextMarker = findNextMarker(marker, from, data, length);
+            if (nextMarker == -1) {
+                break;
+            }
+            BytesReference bytes = data.slice(from, nextMarker - from);
+            try (XContentParser parser = xContent.createParser(registry, bytes)) {
+                consumer.accept(searchRequest, parser);
+            }
+            // move pointers
+            from = nextMarker + 1;
+        }
+    }
+
+    private static int findNextMarker(byte marker, int from, BytesReference data, int length) {
+        for (int i = from; i < length; i++) {
+            if (data.get(i) == marker) {
+                return i;
+            }
+        }
+        if (from != length) {
+            throw new IllegalArgumentException("The msearch request must be terminated by a newline [\n]");
+        }
+        return -1;
+    }
+
+    public static byte[] writeMultiLineFormat(MultiSearchRequest multiSearchRequest, XContent xContent) throws IOException {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        for (SearchRequest request : multiSearchRequest.requests()) {
+            try (XContentBuilder xContentBuilder = XContentBuilder.builder(xContent)) {
+                xContentBuilder.startObject();
+                if (request.indices() != null) {
+                    xContentBuilder.field("index", request.indices());
+                }
+                if (request.indicesOptions() != null && request.indicesOptions() != SearchRequest.DEFAULT_INDICES_OPTIONS) {
+                    if (request.indicesOptions().expandWildcardsOpen() && request.indicesOptions().expandWildcardsClosed()) {
+                        xContentBuilder.field("expand_wildcards", "all");
+                    } else if (request.indicesOptions().expandWildcardsOpen()) {
+                        xContentBuilder.field("expand_wildcards", "open");
+                    } else if (request.indicesOptions().expandWildcardsClosed()) {
+                        xContentBuilder.field("expand_wildcards", "closed");
+                    } else {
+                        xContentBuilder.field("expand_wildcards", "none");
+                    }
+                    xContentBuilder.field("ignore_unavailable", request.indicesOptions().ignoreUnavailable());
+                    xContentBuilder.field("allow_no_indices", request.indicesOptions().allowNoIndices());
+                }
+                if (request.types() != null) {
+                    xContentBuilder.field("types", request.types());
+                }
+                if (request.searchType() != null) {
+                    xContentBuilder.field("search_type", request.searchType().name().toLowerCase(Locale.ROOT));
+                }
+                if (request.requestCache() != null) {
+                    xContentBuilder.field("request_cache", request.requestCache());
+                }
+                if (request.preference() != null) {
+                    xContentBuilder.field("preference", request.preference());
+                }
+                if (request.routing() != null) {
+                    xContentBuilder.field("routing", request.routing());
+                }
+                xContentBuilder.endObject();
+                xContentBuilder.bytes().writeTo(output);
+            }
+            output.write(xContent.streamSeparator());
+            try (XContentBuilder xContentBuilder = XContentBuilder.builder(xContent)) {
+                if (request.source() != null) {
+                    request.source().toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+                } else {
+                    xContentBuilder.startObject();
+                    xContentBuilder.endObject();
+                }
+                xContentBuilder.bytes().writeTo(output);
+            }
+            output.write(xContent.streamSeparator());
+        }
+        return output.toByteArray();
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
index 560379a6ce2..cb30385ecc8 100644
--- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
@@ -24,23 +24,39 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Iterator;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 
 /**
  * A multi search response.
  */
 public class MultiSearchResponse extends ActionResponse implements Iterable, ToXContentObject {
 
+    private static final ParseField RESPONSES = new ParseField(Fields.RESPONSES);
+    private static final ParseField TOOK_IN_MILLIS = new ParseField("took");
+    private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("multi_search",
+            true, a -> new MultiSearchResponse(((List)a[0]).toArray(new Item[0]), (long) a[1]));
+    static {
+        PARSER.declareObjectArray(constructorArg(), (p, c) -> itemFromXContent(p), RESPONSES);
+        PARSER.declareLong(constructorArg(), TOOK_IN_MILLIS);
+    }
+
     /**
      * A search response item, holding the actual search response, or an error message if it failed.
      */
@@ -188,6 +204,45 @@ public class MultiSearchResponse extends ActionResponse implements Iterable failures = new ArrayList<>();
         Clusters clusters = Clusters.EMPTY;
-        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
+        for (Token token = parser.nextToken(); token != Token.END_OBJECT; token = parser.nextToken()) {
+            if (token == Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
                 if (SCROLL_ID.match(currentFieldName)) {
@@ -276,7 +282,7 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
                 } else {
                     parser.skipChildren();
                 }
-            } else if (token == XContentParser.Token.START_OBJECT) {
+            } else if (token == Token.START_OBJECT) {
                 if (SearchHits.Fields.HITS.equals(currentFieldName)) {
                     hits = SearchHits.fromXContent(parser);
                 } else if (Aggregations.AGGREGATIONS_FIELD.equals(currentFieldName)) {
@@ -286,8 +292,8 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
                 } else if (SearchProfileShardResults.PROFILE_FIELD.equals(currentFieldName)) {
                     profile = SearchProfileShardResults.fromXContent(parser);
                 } else if (RestActions._SHARDS_FIELD.match(currentFieldName)) {
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
+                    while ((token = parser.nextToken()) != Token.END_OBJECT) {
+                        if (token == Token.FIELD_NAME) {
                             currentFieldName = parser.currentName();
                         } else if (token.isValue()) {
                             if (RestActions.FAILED_FIELD.match(currentFieldName)) {
@@ -301,9 +307,9 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
                             } else {
                                 parser.skipChildren();
                             }
-                        } else if (token == XContentParser.Token.START_ARRAY) {
+                        } else if (token == Token.START_ARRAY) {
                             if (RestActions.FAILURES_FIELD.match(currentFieldName)) {
-                                while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                                while((token = parser.nextToken()) != Token.END_ARRAY) {
                                     failures.add(ShardSearchFailure.fromXContent(parser));
                                 }
                             } else {
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
index 9dec3be5c1b..371314b990c 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
@@ -76,7 +76,7 @@ public class TransportMultiSearchAction extends HandledTransportAction {
-            try {
-                searchRequest.source(SearchSourceBuilder.fromXContent(parser));
-                multiRequest.add(searchRequest);
-            } catch (IOException e) {
-                throw new ElasticsearchParseException("Exception when parsing search request", e);
-            }
+            searchRequest.source(SearchSourceBuilder.fromXContent(parser));
+            multiRequest.add(searchRequest);
         });
         List requests = multiRequest.requests();
         preFilterShardSize = Math.max(1, preFilterShardSize / (requests.size()+1));
@@ -113,7 +110,7 @@ public class RestMultiSearchAction extends BaseRestHandler {
      * Parses a multi-line {@link RestRequest} body, instantiating a {@link SearchRequest} for each line and applying the given consumer.
      */
     public static void parseMultiLineRequest(RestRequest request, IndicesOptions indicesOptions, boolean allowExplicitIndex,
-            BiConsumer consumer) throws IOException {
+            CheckedBiConsumer consumer) throws IOException {
 
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         String[] types = Strings.splitStringByCommaToArray(request.param("type"));
@@ -123,83 +120,8 @@ public class RestMultiSearchAction extends BaseRestHandler {
         final Tuple sourceTuple = request.contentOrSourceParam();
         final XContent xContent = sourceTuple.v1().xContent();
         final BytesReference data = sourceTuple.v2();
-
-        int from = 0;
-        int length = data.length();
-        byte marker = xContent.streamSeparator();
-        while (true) {
-            int nextMarker = findNextMarker(marker, from, data, length);
-            if (nextMarker == -1) {
-                break;
-            }
-            // support first line with \n
-            if (nextMarker == 0) {
-                from = nextMarker + 1;
-                continue;
-            }
-
-            SearchRequest searchRequest = new SearchRequest();
-            if (indices != null) {
-                searchRequest.indices(indices);
-            }
-            if (indicesOptions != null) {
-                searchRequest.indicesOptions(indicesOptions);
-            }
-            if (types != null && types.length > 0) {
-                searchRequest.types(types);
-            }
-            if (routing != null) {
-                searchRequest.routing(routing);
-            }
-            if (searchType != null) {
-                searchRequest.searchType(searchType);
-            }
-
-            IndicesOptions defaultOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
-
-
-            // now parse the action
-            if (nextMarker - from > 0) {
-                try (XContentParser parser = xContent.createParser(request.getXContentRegistry(), data.slice(from, nextMarker - from))) {
-                    Map source = parser.map();
-                    for (Map.Entry entry : source.entrySet()) {
-                        Object value = entry.getValue();
-                        if ("index".equals(entry.getKey()) || "indices".equals(entry.getKey())) {
-                            if (!allowExplicitIndex) {
-                                throw new IllegalArgumentException("explicit index in multi search is not allowed");
-                            }
-                            searchRequest.indices(nodeStringArrayValue(value));
-                        } else if ("type".equals(entry.getKey()) || "types".equals(entry.getKey())) {
-                            searchRequest.types(nodeStringArrayValue(value));
-                        } else if ("search_type".equals(entry.getKey()) || "searchType".equals(entry.getKey())) {
-                            searchRequest.searchType(nodeStringValue(value, null));
-                        } else if ("request_cache".equals(entry.getKey()) || "requestCache".equals(entry.getKey())) {
-                            searchRequest.requestCache(nodeBooleanValue(value, entry.getKey()));
-                        } else if ("preference".equals(entry.getKey())) {
-                            searchRequest.preference(nodeStringValue(value, null));
-                        } else if ("routing".equals(entry.getKey())) {
-                            searchRequest.routing(nodeStringValue(value, null));
-                        }
-                    }
-                    defaultOptions = IndicesOptions.fromMap(source, defaultOptions);
-                }
-            }
-            searchRequest.indicesOptions(defaultOptions);
-
-            // move pointers
-            from = nextMarker + 1;
-            // now for the body
-            nextMarker = findNextMarker(marker, from, data, length);
-            if (nextMarker == -1) {
-                break;
-            }
-            BytesReference bytes = data.slice(from, nextMarker - from);
-            try (XContentParser parser = xContent.createParser(request.getXContentRegistry(), bytes)) {
-                consumer.accept(searchRequest, parser);
-            }
-            // move pointers
-            from = nextMarker + 1;
-        }
+        MultiSearchRequest.readMultiLineFormat(data, xContent, consumer, indices, indicesOptions, types, routing,
+                searchType, request.getXContentRegistry(), allowExplicitIndex);
     }
 
     @Override
@@ -207,18 +129,6 @@ public class RestMultiSearchAction extends BaseRestHandler {
         return true;
     }
 
-    private static int findNextMarker(byte marker, int from, BytesReference data, int length) {
-        for (int i = from; i < length; i++) {
-            if (data.get(i) == marker) {
-                return i;
-            }
-        }
-        if (from != length) {
-            throw new IllegalArgumentException("The msearch request must be terminated by a newline [\n]");
-        }
-        return -1;
-    }
-
     @Override
     protected Set responseParams() {
         return RESPONSE_PARAMS;
diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index e6de1d859d8..faec42b2587 100644
--- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -19,24 +19,36 @@
 
 package org.elasticsearch.action.search;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestMultiSearchAction;
+import org.elasticsearch.search.Scroll;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.BiConsumer;
 
 import static java.util.Collections.singletonList;
+import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
@@ -202,4 +214,87 @@ public class MultiSearchRequestTests extends ESTestCase {
         return new NamedXContentRegistry(singletonList(new NamedXContentRegistry.Entry(QueryBuilder.class,
                 new ParseField(MatchAllQueryBuilder.NAME), (p, c) -> MatchAllQueryBuilder.fromXContent(p))));
     }
+
+    public void testMultiLineSerialization() throws IOException {
+        int iters = 16;
+        for (int i = 0; i < iters; i++) {
+            // The only formats that support stream separator
+            XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
+            MultiSearchRequest originalRequest = createMultiSearchRequest();
+
+            byte[] originalBytes = MultiSearchRequest.writeMultiLineFormat(originalRequest, xContentType.xContent());
+            MultiSearchRequest parsedRequest = new MultiSearchRequest();
+            CheckedBiConsumer consumer = (r, p) -> {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(p);
+                if (searchSourceBuilder.equals(new SearchSourceBuilder()) == false) {
+                    r.source(searchSourceBuilder);
+                }
+                parsedRequest.add(r);
+            };
+            MultiSearchRequest.readMultiLineFormat(new BytesArray(originalBytes), xContentType.xContent(),
+                    consumer, null, null, null, null, null, xContentRegistry(), true);
+            assertEquals(originalRequest, parsedRequest);
+        }
+    }
+
+    public void testEqualsAndHashcode() throws IOException {
+        checkEqualsAndHashCode(createMultiSearchRequest(), MultiSearchRequestTests::copyRequest, MultiSearchRequestTests::mutate);
+    }
+
+    private static MultiSearchRequest mutate(MultiSearchRequest searchRequest) throws IOException {
+        MultiSearchRequest mutation = copyRequest(searchRequest);
+        List> mutators = new ArrayList<>();
+        mutators.add(() -> mutation.indicesOptions(randomValueOtherThan(searchRequest.indicesOptions(),
+                () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()))));
+        mutators.add(() -> mutation.maxConcurrentSearchRequests(randomIntBetween(1, 32)));
+        mutators.add(() -> mutation.add(createSimpleSearchRequest()));
+        randomFrom(mutators).run();
+        return mutation;
+    }
+
+    private static MultiSearchRequest copyRequest(MultiSearchRequest request) throws IOException {
+        MultiSearchRequest copy = new MultiSearchRequest();
+        if (request.maxConcurrentSearchRequests() > 0) {
+            copy.maxConcurrentSearchRequests(request.maxConcurrentSearchRequests());
+        }
+        copy.indicesOptions(request.indicesOptions());
+        for (SearchRequest searchRequest : request.requests()) {
+            copy.add(searchRequest);
+        }
+        return copy;
+    }
+
+    private static MultiSearchRequest createMultiSearchRequest() throws IOException {
+        int numSearchRequest = randomIntBetween(1, 128);
+        MultiSearchRequest request = new MultiSearchRequest();
+        for (int j = 0; j < numSearchRequest; j++) {
+            SearchRequest searchRequest = createSimpleSearchRequest();
+
+            // scroll is not supported in the current msearch api, so unset it:
+            searchRequest.scroll((Scroll) null);
+
+            // only expand_wildcards, ignore_unavailable and allow_no_indices can be specified from msearch api, so unset other options:
+            IndicesOptions randomlyGenerated = searchRequest.indicesOptions();
+            IndicesOptions msearchDefault = IndicesOptions.strictExpandOpenAndForbidClosed();
+            searchRequest.indicesOptions(IndicesOptions.fromOptions(
+                    randomlyGenerated.ignoreUnavailable(), randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(),
+                    randomlyGenerated.expandWildcardsClosed(), msearchDefault.allowAliasesToMultipleIndices(),
+                    msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases()
+            ));
+
+            request.add(searchRequest);
+        }
+        return request;
+    }
+
+    private static SearchRequest createSimpleSearchRequest() throws IOException {
+        return randomSearchRequest(() -> {
+            // No need to return a very complex SearchSourceBuilder here, that is tested elsewhere
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            searchSourceBuilder.from(randomInt(10));
+            searchSourceBuilder.size(randomIntBetween(20, 100));
+            return searchSourceBuilder;
+        });
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java
new file mode 100644
index 00000000000..e31f593193d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class MultiSearchResponseTests extends ESTestCase {
+
+    public void testFromXContent() throws IOException {
+        for (int runs = 0; runs < 20; runs++) {
+            MultiSearchResponse expected = createTestInstance();
+            XContentType xContentType = randomFrom(XContentType.values());
+            BytesReference shuffled = toShuffledXContent(expected, xContentType, ToXContent.EMPTY_PARAMS, false);
+            XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled);
+            MultiSearchResponse actual = MultiSearchResponse.fromXContext(parser);
+            assertThat(parser.nextToken(), nullValue());
+
+            assertThat(actual.getTook(), equalTo(expected.getTook()));
+            assertThat(actual.getResponses().length, equalTo(expected.getResponses().length));
+            for (int i = 0; i < expected.getResponses().length; i++) {
+                MultiSearchResponse.Item expectedItem = expected.getResponses()[i];
+                MultiSearchResponse.Item actualItem = actual.getResponses()[i];
+                if (expectedItem.isFailure()) {
+                    assertThat(actualItem.getResponse(), nullValue());
+                    assertThat(actualItem.getFailureMessage(), containsString(expectedItem.getFailureMessage()));
+                } else {
+                    assertThat(actualItem.getResponse().toString(), equalTo(expectedItem.getResponse().toString()));
+                    assertThat(actualItem.getFailure(), nullValue());
+                }
+            }
+        }
+    }
+
+    private static MultiSearchResponse createTestInstance() {
+        int numItems = randomIntBetween(0, 128);
+        MultiSearchResponse.Item[] items = new MultiSearchResponse.Item[numItems];
+        for (int i = 0; i < numItems; i++) {
+            if (randomBoolean()) {
+                // Creating a minimal response is OK, because SearchResponse self
+                // is tested elsewhere.
+                long tookInMillis = randomNonNegativeLong();
+                int totalShards = randomIntBetween(1, Integer.MAX_VALUE);
+                int successfulShards = randomIntBetween(0, totalShards);
+                int skippedShards = totalShards - successfulShards;
+                InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
+                SearchResponse.Clusters clusters = new SearchResponse.Clusters(totalShards, successfulShards, skippedShards);
+                SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, totalShards,
+                        successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, clusters);
+                items[i] = new MultiSearchResponse.Item(searchResponse, null);
+            } else {
+                items[i] = new MultiSearchResponse.Item(null, new ElasticsearchException("an error"));
+            }
+        }
+        return new MultiSearchResponse(items, randomNonNegativeLong());
+    }
+
+}
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
index f129a5b15ec..fd797c4340a 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
@@ -77,16 +77,12 @@ public class RestMultiSearchTemplateAction extends BaseRestHandler {
 
         RestMultiSearchAction.parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex,
                 (searchRequest, bytes) -> {
-                    try {
-                        SearchTemplateRequest searchTemplateRequest = RestSearchTemplateAction.parse(bytes);
-                        if (searchTemplateRequest.getScript() != null) {
-                            searchTemplateRequest.setRequest(searchRequest);
-                            multiRequest.add(searchTemplateRequest);
-                        } else {
-                            throw new IllegalArgumentException("Malformed search template");
-                        }
-                    } catch (IOException e) {
-                        throw new ElasticsearchParseException("Exception when parsing search template request", e);
+                    SearchTemplateRequest searchTemplateRequest = RestSearchTemplateAction.parse(bytes);
+                    if (searchTemplateRequest.getScript() != null) {
+                        searchTemplateRequest.setRequest(searchRequest);
+                        multiRequest.add(searchTemplateRequest);
+                    } else {
+                        throw new IllegalArgumentException("Malformed search template");
                     }
                 });
         return multiRequest;

From 0bba2a84383abb2db10a31aac3737c612c13ebb3 Mon Sep 17 00:00:00 2001
From: Clinton Gormley 
Date: Tue, 5 Dec 2017 10:44:02 +0100
Subject: [PATCH 177/297] Update removal_of_types.asciidoc

Corrected  `include_in_type` to `include_type_name`
---
 docs/reference/mapping/removal_of_types.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc
index 006bc789f30..17874f2d3ba 100644
--- a/docs/reference/mapping/removal_of_types.asciidoc
+++ b/docs/reference/mapping/removal_of_types.asciidoc
@@ -274,7 +274,7 @@ Elasticsearch 8.x::
 
 Elasticsearch 9.x::
 
-* The `include_in_type` parameter is removed.
+* The `include_type_name` parameter is removed.
 
 [float]
 === Migrating multi-type indices to single-type

From b447967104be79a449ae6534db9f31eb9ca9d303 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen 
Date: Tue, 5 Dec 2017 11:01:11 +0100
Subject: [PATCH 178/297] removed redundant import

---
 .../elasticsearch/action/search/MultiSearchResponseTests.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java
index e31f593193d..874bea5ff65 100644
--- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchResponseTests.java
@@ -29,7 +29,6 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;

From 1be286c592538fa29cb20e9a456e97f0c8e7bed5 Mon Sep 17 00:00:00 2001
From: Andrew Banchich 
Date: Tue, 5 Dec 2017 05:42:25 -0500
Subject: [PATCH 179/297] [Docs] Grammatical fix in bootstrap-checks.asciidoc
 (#27655)

---
 docs/reference/setup/bootstrap-checks.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
index 58e9867519d..fa099efcf7f 100644
--- a/docs/reference/setup/bootstrap-checks.asciidoc
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -25,7 +25,7 @@ documented individually.
 
 By default, Elasticsearch binds to `localhost` for <> and
 <> communication. This is fine for
-downloading and playing with Elasticsearch, and everyday development but it's
+downloading and playing with Elasticsearch as well as everyday development, but it's
 useless for production systems. To join a cluster, an Elasticsearch node must be
 reachable via transport communication. To join a cluster over an external
 network interface, a node must bind transport to an external interface and not

From eb574425b7c1a78a13c7a8389cb6ca3a25e5ee63 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Tue, 5 Dec 2017 06:58:34 -0500
Subject: [PATCH 180/297] Simplify rejected execution exception

This exception type has several unnecessary constructor overrides so
this commit removes them.

Relates #27664
---
 .../common/util/concurrent/EsExecutors.java   | 10 ++++++---
 .../EsRejectedExecutionException.java         | 22 ++++++-------------
 2 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
index 45d9a208284..057a970470b 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
@@ -246,13 +246,16 @@ public class EsExecutors {
      * waiting if necessary for space to become available.
      */
     static class ForceQueuePolicy implements XRejectedExecutionHandler {
+
         @Override
         public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
             try {
+                // force queue policy should only be used with a scaling queue
+                assert executor.getQueue() instanceof ExecutorScalingQueue;
                 executor.getQueue().put(r);
-            } catch (InterruptedException e) {
-                //should never happen since we never wait
-                throw new EsRejectedExecutionException(e);
+            } catch (final InterruptedException e) {
+                // a scaling queue never blocks so a put to it can never be interrupted
+                throw new AssertionError(e);
             }
         }
 
@@ -260,6 +263,7 @@ public class EsExecutors {
         public long rejected() {
             return 0;
         }
+
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java
index 01fbbac725b..f05b9d10926 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java
@@ -27,29 +27,20 @@ import org.elasticsearch.rest.RestStatus;
 import java.io.IOException;
 
 public class EsRejectedExecutionException extends ElasticsearchException {
+
     private final boolean isExecutorShutdown;
 
-    public EsRejectedExecutionException(String message, boolean isExecutorShutdown, Object... args) {
-        super(message, args);
+    public EsRejectedExecutionException(String message, boolean isExecutorShutdown) {
+        super(message, isExecutorShutdown);
         this.isExecutorShutdown = isExecutorShutdown;
     }
 
-    public EsRejectedExecutionException(String message, Object... args) {
-        this(message, false, args);
-    }
-
-    public EsRejectedExecutionException(String message, boolean isExecutorShutdown) {
-        this(message, isExecutorShutdown, new Object[0]);
+    public EsRejectedExecutionException(String message) {
+        this(message, false);
     }
 
     public EsRejectedExecutionException() {
-        super((String)null);
-        this.isExecutorShutdown = false;
-    }
-
-    public EsRejectedExecutionException(Throwable e) {
-        super(null, e);
-        this.isExecutorShutdown = false;
+        this(null, false);
     }
 
     @Override
@@ -79,4 +70,5 @@ public class EsRejectedExecutionException extends ElasticsearchException {
     public boolean isExecutorShutdown() {
         return isExecutorShutdown;
     }
+
 }

From 83c8daa5e892b20316b2a0f610a625b6c8e96053 Mon Sep 17 00:00:00 2001
From: Boaz Leskes 
Date: Tue, 5 Dec 2017 13:00:42 +0100
Subject: [PATCH 181/297] FullClusterRestartIT.testRecovery - add more info on
 failure

---
 .../org/elasticsearch/upgrades/FullClusterRestartIT.java     | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
index 568ae43c323..d357a228c43 100644
--- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -696,8 +696,9 @@ public class FullClusterRestartIT extends ESRestTestCase {
                     fail("expected version to be one of [" + currentLuceneVersion + "," + bwcLuceneVersion + "] but was " + line);
                 }
             }
-            assertNotEquals("expected at least 1 current segment after translog recovery", 0, numCurrentVersion);
-            assertNotEquals("expected at least 1 old segment", 0, numBwcVersion);}
+            assertNotEquals("expected at least 1 current segment after translog recovery. segments:\n" + segmentsResponse,
+                0, numCurrentVersion);
+            assertNotEquals("expected at least 1 old segment. segments:\n" + segmentsResponse, 0, numBwcVersion);}
         }
     }
 

From 144e1698cf23d555ae21af8ab91f4451e0876679 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Tue, 5 Dec 2017 07:09:53 -0500
Subject: [PATCH 182/297] Do not pass executor shutdown to super constructor

The main constructor for rejected execution exception its executor
shutdown constructor parameter to the super constructor where it would
be used as a formatting parameter. This is a mistake so this commit
fixes this issue.
---
 .../common/util/concurrent/EsRejectedExecutionException.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java
index f05b9d10926..a38bbf452b7 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsRejectedExecutionException.java
@@ -31,7 +31,7 @@ public class EsRejectedExecutionException extends ElasticsearchException {
     private final boolean isExecutorShutdown;
 
     public EsRejectedExecutionException(String message, boolean isExecutorShutdown) {
-        super(message, isExecutorShutdown);
+        super(message);
         this.isExecutorShutdown = isExecutorShutdown;
     }
 

From 42a4ad35da47712039a89a54514b4fa6bbce03b7 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Tue, 5 Dec 2017 07:45:40 -0500
Subject: [PATCH 183/297] Add node name to thread pool executor name

This commit adds the node name to the names of thread pool executors so
that the node name is visible in rejected execution exception messages.

Relates #27663
---
 .../service/ClusterApplierService.java        |  7 +-
 .../cluster/service/MasterService.java        |  7 +-
 .../util/concurrent/EsThreadPoolExecutor.java | 21 ++++-
 .../QueueResizingEsThreadPoolExecutor.java    | 47 ++++------
 .../discovery/zen/UnicastZenPing.java         | 13 +--
 .../AutoQueueAdjustingExecutorBuilder.java    | 21 +++--
 .../threadpool/FixedExecutorBuilder.java      |  3 +-
 .../threadpool/ScalingExecutorBuilder.java    |  9 +-
 .../cluster/service/TaskExecutorTests.java    |  2 +-
 .../util/concurrent/EsExecutorsTests.java     | 31 ++++---
 .../concurrent/EsThreadPoolExecutorTests.java | 87 +++++++++++++++++++
 .../concurrent/PrioritizedExecutorsTests.java | 19 ++--
 .../discovery/zen/UnicastZenPingTests.java    |  3 +-
 .../file/FileBasedDiscoveryPlugin.java        | 16 ++--
 .../test/InternalTestCluster.java             |  4 +-
 15 files changed, 205 insertions(+), 85 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutorTests.java

diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
index 13c2e50eba2..9914ee2577a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
@@ -133,8 +133,11 @@ public class ClusterApplierService extends AbstractLifecycleComponent implements
         Objects.requireNonNull(nodeConnectionsService, "please set the node connection service before starting");
         Objects.requireNonNull(state.get(), "please set initial state before starting");
         addListener(localNodeMasterListeners);
-        threadPoolExecutor = EsExecutors.newSinglePrioritizing(CLUSTER_UPDATE_THREAD_NAME,
-            daemonThreadFactory(settings, CLUSTER_UPDATE_THREAD_NAME), threadPool.getThreadContext(), threadPool.scheduler());
+        threadPoolExecutor = EsExecutors.newSinglePrioritizing(
+                nodeName() + "/" + CLUSTER_UPDATE_THREAD_NAME,
+                daemonThreadFactory(settings, CLUSTER_UPDATE_THREAD_NAME),
+                threadPool.getThreadContext(),
+                threadPool.scheduler());
     }
 
     class UpdateTask extends SourcePrioritizedRunnable implements Function {
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/MasterService.java b/core/src/main/java/org/elasticsearch/cluster/service/MasterService.java
index a5f71dc48b8..6858866d2dc 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/MasterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/MasterService.java
@@ -104,8 +104,11 @@ public class MasterService extends AbstractLifecycleComponent {
     protected synchronized void doStart() {
         Objects.requireNonNull(clusterStatePublisher, "please set a cluster state publisher before starting");
         Objects.requireNonNull(clusterStateSupplier, "please set a cluster state supplier before starting");
-        threadPoolExecutor = EsExecutors.newSinglePrioritizing(MASTER_UPDATE_THREAD_NAME,
-            daemonThreadFactory(settings, MASTER_UPDATE_THREAD_NAME), threadPool.getThreadContext(), threadPool.scheduler());
+        threadPoolExecutor = EsExecutors.newSinglePrioritizing(
+                nodeName() + "/" + MASTER_UPDATE_THREAD_NAME,
+                daemonThreadFactory(settings, MASTER_UPDATE_THREAD_NAME),
+                threadPool.getThreadContext(),
+                threadPool.scheduler());
         taskBatcher = new Batcher(logger, threadPoolExecutor);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java
index a1ac182b8dc..8bbf0a59ee0 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutor.java
@@ -37,7 +37,11 @@ public class EsThreadPoolExecutor extends ThreadPoolExecutor {
     /**
      * Name used in error reporting.
      */
-    protected final String name;
+    private final String name;
+
+    final String getName() {
+        return name;
+    }
 
     EsThreadPoolExecutor(String name, int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
             BlockingQueue workQueue, ThreadFactory threadFactory, ThreadContext contextHolder) {
@@ -138,15 +142,16 @@ public class EsThreadPoolExecutor extends ThreadPoolExecutor {
     }
 
     @Override
-    public String toString() {
+    public final String toString() {
         StringBuilder b = new StringBuilder();
         b.append(getClass().getSimpleName()).append('[');
-        b.append(name).append(", ");
+        b.append("name = ").append(name).append(", ");
         if (getQueue() instanceof SizeBlockingQueue) {
             @SuppressWarnings("rawtypes")
             SizeBlockingQueue queue = (SizeBlockingQueue) getQueue();
             b.append("queue capacity = ").append(queue.capacity()).append(", ");
         }
+        appendThreadPoolExecutorDetails(b);
         /*
          * ThreadPoolExecutor has some nice information in its toString but we
          * can't get at it easily without just getting the toString.
@@ -155,6 +160,16 @@ public class EsThreadPoolExecutor extends ThreadPoolExecutor {
         return b.toString();
     }
 
+    /**
+     * Append details about this thread pool to the specified {@link StringBuilder}. All details should be appended as key/value pairs in
+     * the form "%s = %s, "
+     *
+     * @param sb the {@link StringBuilder} to append to
+     */
+    protected void appendThreadPoolExecutorDetails(final StringBuilder sb) {
+
+    }
+
     protected Runnable wrapRunnable(Runnable command) {
         return contextHolder.preserveContext(command);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/QueueResizingEsThreadPoolExecutor.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/QueueResizingEsThreadPoolExecutor.java
index 8062d5510c7..e929192b5dd 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/QueueResizingEsThreadPoolExecutor.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/QueueResizingEsThreadPoolExecutor.java
@@ -22,21 +22,16 @@ package org.elasticsearch.common.util.concurrent;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.common.ExponentiallyWeightedMovingAverage;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.ResizableBlockingQueue;
 
 import java.util.Locale;
-import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
 import java.util.function.Supplier;
-import java.util.stream.Stream;
 
 /**
  * An extension to thread pool executor, which automatically adjusts the queue size of the
@@ -80,8 +75,8 @@ public final class QueueResizingEsThreadPoolExecutor extends EsThreadPoolExecuto
         this.maxQueueSize = maxQueueSize;
         this.targetedResponseTimeNanos = targetedResponseTime.getNanos();
         this.executionEWMA = new ExponentiallyWeightedMovingAverage(EWMA_ALPHA, 0);
-        logger.debug("thread pool [{}] will adjust queue by [{}] when determining automatic queue size",
-                name, QUEUE_ADJUSTMENT_AMOUNT);
+        logger.debug(
+                "thread pool [{}] will adjust queue by [{}] when determining automatic queue size", getName(), QUEUE_ADJUSTMENT_AMOUNT);
     }
 
     @Override
@@ -180,7 +175,7 @@ public final class QueueResizingEsThreadPoolExecutor extends EsThreadPoolExecuto
                     final long avgTaskTime = totalNanos / tasksPerFrame;
                     logger.debug("[{}]: there were [{}] tasks in [{}], avg task time [{}], EWMA task execution [{}], " +
                                     "[{} tasks/s], optimal queue is [{}], current capacity [{}]",
-                            name,
+                            getName(),
                             tasksPerFrame,
                             TimeValue.timeValueNanos(totalRuntime),
                             TimeValue.timeValueNanos(avgTaskTime),
@@ -196,7 +191,7 @@ public final class QueueResizingEsThreadPoolExecutor extends EsThreadPoolExecuto
                 final int newCapacity =
                         workQueue.adjustCapacity(desiredQueueSize, QUEUE_ADJUSTMENT_AMOUNT, minQueueSize, maxQueueSize);
                 if (oldCapacity != newCapacity && logger.isDebugEnabled()) {
-                    logger.debug("adjusted [{}] queue size by [{}], old capacity: [{}], new capacity: [{}]", name,
+                    logger.debug("adjusted [{}] queue size by [{}], old capacity: [{}], new capacity: [{}]", getName(),
                             newCapacity > oldCapacity ? QUEUE_ADJUSTMENT_AMOUNT : -QUEUE_ADJUSTMENT_AMOUNT,
                             oldCapacity, newCapacity);
                 }
@@ -205,7 +200,7 @@ public final class QueueResizingEsThreadPoolExecutor extends EsThreadPoolExecuto
                 logger.warn((Supplier) () -> new ParameterizedMessage(
                                 "failed to calculate optimal queue size for [{}] thread pool, " +
                                 "total frame time [{}ns], tasks [{}], task execution time [{}ns]",
-                                name, totalRuntime, tasksPerFrame, totalNanos),
+                                getName(), totalRuntime, tasksPerFrame, totalNanos),
                         e);
             } finally {
                 // Finally, decrement the task count and time back to their starting values. We
@@ -224,7 +219,8 @@ public final class QueueResizingEsThreadPoolExecutor extends EsThreadPoolExecuto
                     // - Adjustment happens and we decrement the tasks by 10, taskCount is now 15
                     // - Since taskCount will now be incremented forever, it will never be 10 again,
                     //   so there will be no further adjustments
-                    logger.debug("[{}]: too many incoming tasks while queue size adjustment occurs, resetting measurements to 0", name);
+                    logger.debug(
+                            "[{}]: too many incoming tasks while queue size adjustment occurs, resetting measurements to 0", getName());
                     totalTaskNanos.getAndSet(1);
                     taskCount.getAndSet(0);
                     startNs = System.nanoTime();
@@ -237,26 +233,13 @@ public final class QueueResizingEsThreadPoolExecutor extends EsThreadPoolExecuto
     }
 
     @Override
-    public String toString() {
-        StringBuilder b = new StringBuilder();
-        b.append(getClass().getSimpleName()).append('[');
-        b.append(name).append(", ");
-
-        @SuppressWarnings("rawtypes")
-        ResizableBlockingQueue queue = (ResizableBlockingQueue) getQueue();
-
-        b.append("queue capacity = ").append(getCurrentCapacity()).append(", ");
-        b.append("min queue capacity = ").append(minQueueSize).append(", ");
-        b.append("max queue capacity = ").append(maxQueueSize).append(", ");
-        b.append("frame size = ").append(tasksPerFrame).append(", ");
-        b.append("targeted response rate = ").append(TimeValue.timeValueNanos(targetedResponseTimeNanos)).append(", ");
-        b.append("task execution EWMA = ").append(TimeValue.timeValueNanos((long)executionEWMA.getAverage())).append(", ");
-        b.append("adjustment amount = ").append(QUEUE_ADJUSTMENT_AMOUNT).append(", ");
-        /*
-         * ThreadPoolExecutor has some nice information in its toString but we
-         * can't get at it easily without just getting the toString.
-         */
-        b.append(super.toString()).append(']');
-        return b.toString();
+    protected void appendThreadPoolExecutorDetails(StringBuilder sb) {
+        sb.append("min queue capacity = ").append(minQueueSize).append(", ");
+        sb.append("max queue capacity = ").append(maxQueueSize).append(", ");
+        sb.append("frame size = ").append(tasksPerFrame).append(", ");
+        sb.append("targeted response rate = ").append(TimeValue.timeValueNanos(targetedResponseTimeNanos)).append(", ");
+        sb.append("task execution EWMA = ").append(TimeValue.timeValueNanos((long) executionEWMA.getAverage())).append(", ");
+        sb.append("adjustment amount = ").append(QUEUE_ADJUSTMENT_AMOUNT).append(", ");
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
index 06269706e0d..54cdb7caeaa 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
@@ -167,12 +167,13 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
 
         final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[unicast_connect]");
         unicastZenPingExecutorService = EsExecutors.newScaling(
-            "unicast_connect",
-            0, concurrentConnects,
-            60,
-            TimeUnit.SECONDS,
-            threadFactory,
-            threadPool.getThreadContext());
+                nodeName() + "/" + "unicast_connect",
+                0,
+                concurrentConnects,
+                60,
+                TimeUnit.SECONDS,
+                threadFactory,
+                threadPool.getThreadContext());
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/threadpool/AutoQueueAdjustingExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/AutoQueueAdjustingExecutorBuilder.java
index 265e544d281..ec9d95c722d 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/AutoQueueAdjustingExecutorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/AutoQueueAdjustingExecutorBuilder.java
@@ -19,23 +19,14 @@
 
 package org.elasticsearch.threadpool;
 
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.SizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.threadpool.ExecutorBuilder;
-import org.elasticsearch.common.util.concurrent.QueueResizingEsThreadPoolExecutor;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
@@ -121,8 +112,16 @@ public final class AutoQueueAdjustingExecutorBuilder extends ExecutorBuilder {
                 try {
@@ -360,7 +370,8 @@ public class EsExecutorsTests extends ESTestCase {
         int queue = between(0, 100);
         final CountDownLatch latch = new CountDownLatch(1);
         final CountDownLatch executed = new CountDownLatch(1);
-        EsThreadPoolExecutor executor = EsExecutors.newFixed(getTestName(), pool, queue, EsExecutors.daemonThreadFactory("dummy"), threadContext);
+        EsThreadPoolExecutor executor =
+                EsExecutors.newFixed(getName(), pool, queue, EsExecutors.daemonThreadFactory("dummy"), threadContext);
         try {
             Runnable r = () -> {
                 latch.countDown();
@@ -379,6 +390,6 @@ public class EsExecutorsTests extends ESTestCase {
             latch.countDown();
             terminate(executor);
         }
-
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutorTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutorTests.java
new file mode 100644
index 00000000000..9b9aa50bd16
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutorTests.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.util.concurrent;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasToString;
+
+public class EsThreadPoolExecutorTests extends ESSingleNodeTestCase {
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder()
+                .put("node.name", "es-thread-pool-executor-tests")
+                .put("thread_pool.bulk.size", 1)
+                .put("thread_pool.bulk.queue_size", 0)
+                .put("thread_pool.search.size", 1)
+                .put("thread_pool.search.queue_size", 1)
+                .build();
+    }
+
+    public void testRejectedExecutionExceptionContainsNodeName() {
+        // we test a fixed and an auto-queue executor but not scaling since it does not reject
+        runThreadPoolExecutorTest(1, ThreadPool.Names.BULK);
+        runThreadPoolExecutorTest(2, ThreadPool.Names.SEARCH);
+
+    }
+
+    private void runThreadPoolExecutorTest(final int fill, final String executor) {
+        final CountDownLatch latch = new CountDownLatch(fill);
+        for (int i = 0; i < fill; i++) {
+            node().injector().getInstance(ThreadPool.class).executor(executor).execute(() -> {
+                try {
+                    latch.await();
+                } catch (final InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+        }
+
+        final AtomicBoolean rejected = new AtomicBoolean();
+        node().injector().getInstance(ThreadPool.class).executor(executor).execute(new AbstractRunnable() {
+            @Override
+            public void onFailure(Exception e) {
+
+            }
+
+            @Override
+            public void onRejection(Exception e) {
+                rejected.set(true);
+                assertThat(e, hasToString(containsString("name = es-thread-pool-executor-tests/" + executor + ", ")));
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+
+            }
+        });
+
+        latch.countDown();
+        assertTrue(rejected.get());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java
index 17b43a079dc..1eacb4cb18c 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java
@@ -45,6 +45,10 @@ public class PrioritizedExecutorsTests extends ESTestCase {
 
     private final ThreadContext holder = new ThreadContext(Settings.EMPTY);
 
+    private String getName() {
+        return getClass().getName() + "/" + getTestName();
+    }
+
     public void testPriorityQueue() throws Exception {
         PriorityBlockingQueue queue = new PriorityBlockingQueue<>();
         List priorities = Arrays.asList(Priority.values());
@@ -65,7 +69,8 @@ public class PrioritizedExecutorsTests extends ESTestCase {
     }
 
     public void testSubmitPrioritizedExecutorWithRunnables() throws Exception {
-        ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder, null);
+        ExecutorService executor =
+                EsExecutors.newSinglePrioritizing(getName(), EsExecutors.daemonThreadFactory(getTestName()), holder, null);
         List results = new ArrayList<>(8);
         CountDownLatch awaitingLatch = new CountDownLatch(1);
         CountDownLatch finishedLatch = new CountDownLatch(8);
@@ -94,7 +99,8 @@ public class PrioritizedExecutorsTests extends ESTestCase {
     }
 
     public void testExecutePrioritizedExecutorWithRunnables() throws Exception {
-        ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder, null);
+        ExecutorService executor =
+                EsExecutors.newSinglePrioritizing(getName(), EsExecutors.daemonThreadFactory(getTestName()), holder, null);
         List results = new ArrayList<>(8);
         CountDownLatch awaitingLatch = new CountDownLatch(1);
         CountDownLatch finishedLatch = new CountDownLatch(8);
@@ -123,7 +129,8 @@ public class PrioritizedExecutorsTests extends ESTestCase {
     }
 
     public void testSubmitPrioritizedExecutorWithCallables() throws Exception {
-        ExecutorService executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder, null);
+        ExecutorService executor =
+                EsExecutors.newSinglePrioritizing(getName(), EsExecutors.daemonThreadFactory(getTestName()), holder, null);
         List results = new ArrayList<>(8);
         CountDownLatch awaitingLatch = new CountDownLatch(1);
         CountDownLatch finishedLatch = new CountDownLatch(8);
@@ -182,7 +189,8 @@ public class PrioritizedExecutorsTests extends ESTestCase {
 
     public void testTimeout() throws Exception {
         ScheduledExecutorService timer = Executors.newSingleThreadScheduledExecutor(EsExecutors.daemonThreadFactory(getTestName()));
-        PrioritizedEsThreadPoolExecutor executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder, timer);
+        PrioritizedEsThreadPoolExecutor executor =
+                EsExecutors.newSinglePrioritizing(getName(), EsExecutors.daemonThreadFactory(getTestName()), holder, timer);
         final CountDownLatch invoked = new CountDownLatch(1);
         final CountDownLatch block = new CountDownLatch(1);
         executor.execute(new Runnable() {
@@ -245,7 +253,8 @@ public class PrioritizedExecutorsTests extends ESTestCase {
         ThreadPool threadPool = new TestThreadPool("test");
         final ScheduledThreadPoolExecutor timer = (ScheduledThreadPoolExecutor) threadPool.scheduler();
         final AtomicBoolean timeoutCalled = new AtomicBoolean();
-        PrioritizedEsThreadPoolExecutor executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder, timer);
+        PrioritizedEsThreadPoolExecutor executor =
+                EsExecutors.newSinglePrioritizing(getName(), EsExecutors.daemonThreadFactory(getTestName()), holder, timer);
         final CountDownLatch invoked = new CountDownLatch(1);
         executor.execute(new Runnable() {
                              @Override
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
index 853294de186..d40d558d20b 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
@@ -112,7 +112,8 @@ public class UnicastZenPingTests extends ESTestCase {
         threadPool = new TestThreadPool(getClass().getName());
         final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory("[" + getClass().getName() + "]");
         executorService =
-            EsExecutors.newScaling(getClass().getName(), 0, 2, 60, TimeUnit.SECONDS, threadFactory, threadPool.getThreadContext());
+            EsExecutors.newScaling(
+                    getClass().getName() + "/" + getTestName(), 0, 2, 60, TimeUnit.SECONDS, threadFactory, threadPool.getThreadContext());
         closeables = new Stack<>();
     }
 
diff --git a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java
index b5d16a547d5..a8f3337d50d 100644
--- a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java
+++ b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java
@@ -29,11 +29,11 @@ import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.discovery.zen.UnicastZenPing;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
@@ -78,13 +78,13 @@ public class FileBasedDiscoveryPlugin extends Plugin implements DiscoveryPlugin
         final int concurrentConnects = UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING.get(settings);
         final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[file_based_discovery_resolve]");
         fileBasedDiscoveryExecutorService = EsExecutors.newScaling(
-            "file_based_discovery_resolve",
-            0,
-            concurrentConnects,
-            60,
-            TimeUnit.SECONDS,
-            threadFactory,
-            threadPool.getThreadContext());
+                Node.NODE_NAME_SETTING.get(settings) + "/" + "file_based_discovery_resolve",
+                0,
+                concurrentConnects,
+                60,
+                TimeUnit.SECONDS,
+                threadFactory,
+                threadPool.getThreadContext());
 
         return Collections.emptyList();
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index ea9b17f10e3..0b001de69cf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -138,8 +138,8 @@ import static org.elasticsearch.discovery.DiscoverySettings.INITIAL_STATE_TIMEOU
 import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
 import static org.elasticsearch.test.ESTestCase.assertBusy;
 import static org.elasticsearch.test.ESTestCase.awaitBusy;
-import static org.elasticsearch.test.ESTestCase.randomFrom;
 import static org.elasticsearch.test.ESTestCase.getTestTransportType;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -349,7 +349,7 @@ public final class InternalTestCluster extends TestCluster {
         // always reduce this - it can make tests really slow
         builder.put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.getKey(), TimeValue.timeValueMillis(RandomNumbers.randomIntBetween(random, 20, 50)));
         defaultSettings = builder.build();
-        executor = EsExecutors.newScaling("test runner", 0, Integer.MAX_VALUE, 0, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory("test_" + clusterName), new ThreadContext(Settings.EMPTY));
+        executor = EsExecutors.newScaling("internal_test_cluster_executor", 0, Integer.MAX_VALUE, 0, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory("test_" + clusterName), new ThreadContext(Settings.EMPTY));
     }
 
     @Override

From 14e972fa424b07df50ddfd462d3b0f14d02205ec Mon Sep 17 00:00:00 2001
From: Glen Smith 
Date: Tue, 5 Dec 2017 06:30:22 -0700
Subject: [PATCH 184/297] [Docs] Fix parameter name (#27656)


From 5b03e3b53d235befcf7362ae26ec75ea9b3e923b Mon Sep 17 00:00:00 2001
From: David Turner 
Date: Tue, 5 Dec 2017 16:19:33 +0000
Subject: [PATCH 185/297] Fix up tests now that GeoDistance.*.calculate works
 (#27541)

This resolves a longstanding @AwaitsFix
---
 .../common/geo/GeoDistanceTests.java          | 77 +++++++++++++------
 1 file changed, 55 insertions(+), 22 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
index 6624a91a6c2..c2e62277da7 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
@@ -81,29 +81,62 @@ public class GeoDistanceTests extends ESTestCase {
         assertThat(GeoUtils.rectangleContainsPoint(box, 0, -178), equalTo(false));
     }
 
-    /**
-     * The old plane calculation in 1.x/2.x incorrectly computed the plane distance in decimal degrees. This test is
-     * well intended but bogus. todo: fix w/ new plane distance calculation
-     * note: plane distance error varies by latitude so the test will need to correctly estimate expected error
-     */
-    @AwaitsFix(bugUrl = "old plane calculation incorrectly computed everything in degrees. fix this bogus test")
-    public void testArcDistanceVsPlaneInEllipsis() {
-        GeoPoint centre = new GeoPoint(48.8534100, 2.3488000);
-        GeoPoint northernPoint = new GeoPoint(48.8801108681, 2.35152032666);
-        GeoPoint westernPoint = new GeoPoint(48.85265, 2.308896);
+    private static double arcDistance(GeoPoint p1, GeoPoint p2) {
+        return GeoDistance.ARC.calculate(p1.lat(), p1.lon(), p2.lat(), p2.lon(), DistanceUnit.METERS);
+    }
 
-        // With GeoDistance.ARC both the northern and western points are within the 4km range
-        assertThat(GeoDistance.ARC.calculate(centre.lat(), centre.lon(), northernPoint.lat(),
-                northernPoint.lon(), DistanceUnit.KILOMETERS), lessThan(4D));
-        assertThat(GeoDistance.ARC.calculate(centre.lat(), centre.lon(), westernPoint.lat(),
-                westernPoint.lon(), DistanceUnit.KILOMETERS), lessThan(4D));
+    private static double planeDistance(GeoPoint p1, GeoPoint p2) {
+        return GeoDistance.PLANE.calculate(p1.lat(), p1.lon(), p2.lat(), p2.lon(), DistanceUnit.METERS);
+    }
 
-        // With GeoDistance.PLANE, only the northern point is within the 4km range,
-        // the western point is outside of the range due to the simple math it employs,
-        // meaning results will appear elliptical
-        assertThat(GeoDistance.PLANE.calculate(centre.lat(), centre.lon(), northernPoint.lat(),
-                northernPoint.lon(), DistanceUnit.KILOMETERS), lessThan(4D));
-        assertThat(GeoDistance.PLANE.calculate(centre.lat(), centre.lon(), westernPoint.lat(),
-                westernPoint.lon(), DistanceUnit.KILOMETERS), greaterThan(4D));
+    public void testArcDistanceVsPlane() {
+        // sameLongitude and sameLatitude are both 90 degrees away from basePoint along great circles
+        final GeoPoint basePoint = new GeoPoint(45, 90);
+        final GeoPoint sameLongitude = new GeoPoint(-45, 90);
+        final GeoPoint sameLatitude = new GeoPoint(45, -90);
+
+        double sameLongitudeArcDistance = arcDistance(basePoint, sameLongitude);
+        double sameLatitudeArcDistance = arcDistance(basePoint, sameLatitude);
+        double sameLongitudePlaneDistance = planeDistance(basePoint, sameLongitude);
+        double sameLatitudePlaneDistance = planeDistance(basePoint, sameLatitude);
+
+        // GeoDistance.PLANE measures the distance along a straight line in
+        // (lat, long) space so agrees with GeoDistance.ARC along a line of
+        // constant longitude but takes a longer route if there is east/west
+        // movement.
+
+        assertThat("Arc and plane should agree on sameLongitude",
+            Math.abs(sameLongitudeArcDistance - sameLongitudePlaneDistance), lessThan(0.001));
+
+        assertThat("Arc and plane should disagree on sameLatitude (by >4000km)",
+            sameLatitudePlaneDistance - sameLatitudeArcDistance, greaterThan(4.0e6));
+
+        // GeoDistance.ARC calculates the great circle distance (on a sphere) so these should agree as they're both 90 degrees
+        assertThat("Arc distances should agree", Math.abs(sameLongitudeArcDistance - sameLatitudeArcDistance), lessThan(0.001));
+    }
+
+    public void testArcDistanceVsPlaneAccuracy() {
+        // These points only differ by a few degrees so the calculation methods
+        // should match more closely. Check that the deviation is small enough,
+        // but not too small.
+
+        // The biggest deviations are away from the equator and the poles so pick a suitably troublesome latitude.
+        GeoPoint basePoint = new GeoPoint(randomDoubleBetween(30.0, 60.0, true), randomDoubleBetween(-180.0, 180.0, true));
+        GeoPoint sameLongitude = new GeoPoint(randomDoubleBetween(-90.0, 90.0, true), basePoint.lon());
+        GeoPoint sameLatitude = new GeoPoint(basePoint.lat(), basePoint.lon() + randomDoubleBetween(4.0, 10.0, true));
+
+        double sameLongitudeArcDistance = arcDistance(basePoint, sameLongitude);
+        double sameLatitudeArcDistance = arcDistance(basePoint, sameLatitude);
+        double sameLongitudePlaneDistance = planeDistance(basePoint, sameLongitude);
+        double sameLatitudePlaneDistance = planeDistance(basePoint, sameLatitude);
+
+        assertThat("Arc and plane should agree [" + basePoint + "] to [" + sameLongitude + "] (within 1cm)",
+            Math.abs(sameLongitudeArcDistance - sameLongitudePlaneDistance), lessThan(0.01));
+
+        assertThat("Arc and plane should very roughly agree [" + basePoint + "] to [" + sameLatitude + "]",
+            sameLatitudePlaneDistance - sameLatitudeArcDistance, lessThan(600.0));
+
+        assertThat("Arc and plane should disagree by some margin [" + basePoint + "] to [" + sameLatitude + "]",
+            sameLatitudePlaneDistance - sameLatitudeArcDistance, greaterThan(15.0));
     }
 }

From 8bcf5393f2f200bba2480bc11e9cbd798e7a5acd Mon Sep 17 00:00:00 2001
From: Nicholas Knize 
Date: Thu, 16 Nov 2017 13:35:30 -0600
Subject: [PATCH 186/297] [Geo] Add Well Known Text (WKT) Parsing Support to
 ShapeBuilders

This commit adds WKT support to Geo ShapeBuilders.

This supports the following format:

POINT (30 10)
LINESTRING (30 10, 10 30, 40 40)
BBOX (-10, 10, 10, -10)
POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))
POLYGON ((35 10, 45 45, 15 40, 10 20, 35 10), (20 30, 35 35, 30 20, 20 30))
MULTIPOINT ((10 40), (40 30), (20 20), (30 10))
MULTIPOINT (10 40, 40 30, 20 20, 30 10)
MULTILINESTRING ((10 10, 20 20, 10 40),(40 40, 30 30, 40 20, 30 10))
MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), ((15 5, 40 10, 10 20, 5 10, 15 5)))
MULTIPOLYGON (((40 40, 20 45, 45 30, 40 40)), ((20 35, 10 30, 10 10, 30 5, 45 20, 20 35), (30 20, 20 15, 20 25, 30 20)))
GEOMETRYCOLLECTION (POINT (30 10), MULTIPOINT ((10 40), (40 30), (20 20), (30 10)))

closes #9120
---
 .../common/geo/GeoShapeType.java              |  17 +
 .../common/geo/builders/CircleBuilder.java    |   5 +
 .../common/geo/builders/EnvelopeBuilder.java  |  23 ++
 .../builders/GeometryCollectionBuilder.java   |  18 +
 .../geo/builders/MultiLineStringBuilder.java  |  21 +-
 .../geo/builders/MultiPolygonBuilder.java     |  32 ++
 .../common/geo/builders/PolygonBuilder.java   |  13 +
 .../common/geo/builders/ShapeBuilder.java     |  42 +++
 .../common/geo/parsers/GeoWKTParser.java      | 321 ++++++++++++++++++
 .../common/geo/parsers/ShapeParser.java       |   2 +
 .../common/geo/BaseGeoParsingTestCase.java    |  42 +++
 .../common/geo/GeoJsonShapeParserTests.java   |  27 +-
 .../common/geo/GeoWKTShapeParserTests.java    | 255 ++++++++++++++
 .../mapping/types/geo-shape.asciidoc          | 171 ++++++++--
 14 files changed, 933 insertions(+), 56 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java
 create mode 100644 core/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java

diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoShapeType.java b/core/src/main/java/org/elasticsearch/common/geo/GeoShapeType.java
index f8030296940..9eb1fa9a3f4 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoShapeType.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoShapeType.java
@@ -241,6 +241,11 @@ public enum GeoShapeType {
             }
             return coordinates;
         }
+
+        @Override
+        public String wktName() {
+            return BBOX;
+        }
     },
     CIRCLE("circle") {
         @Override
@@ -273,11 +278,13 @@ public enum GeoShapeType {
 
     private final String shapename;
     private static Map shapeTypeMap = new HashMap<>();
+    private static final String BBOX = "BBOX";
 
     static {
         for (GeoShapeType type : values()) {
             shapeTypeMap.put(type.shapename, type);
         }
+        shapeTypeMap.put(ENVELOPE.wktName().toLowerCase(Locale.ROOT), ENVELOPE);
     }
 
     GeoShapeType(String shapename) {
@@ -300,6 +307,11 @@ public enum GeoShapeType {
                                             ShapeBuilder.Orientation orientation, boolean coerce);
     abstract CoordinateNode validate(CoordinateNode coordinates, boolean coerce);
 
+    /** wkt shape name */
+    public String wktName() {
+        return this.shapename;
+    }
+
     public static List getShapeWriteables() {
         List namedWriteables = new ArrayList<>();
         namedWriteables.add(new Entry(ShapeBuilder.class, PointBuilder.TYPE.shapeName(), PointBuilder::new));
@@ -313,4 +325,9 @@ public enum GeoShapeType {
         namedWriteables.add(new Entry(ShapeBuilder.class, GeometryCollectionBuilder.TYPE.shapeName(), GeometryCollectionBuilder::new));
         return namedWriteables;
     }
+
+    @Override
+    public String toString() {
+        return this.shapename;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java
index 108e66d9150..ecc33b94ae4 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/CircleBuilder.java
@@ -168,6 +168,11 @@ public class CircleBuilder extends ShapeBuilder {
         return TYPE;
     }
 
+    @Override
+    public String toWKT() {
+        throw new UnsupportedOperationException("The WKT spec does not support CIRCLE geometry");
+    }
+
     @Override
     public int hashCode() {
         return Objects.hash(center, radius, unit.ordinal());
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java
index b352aa1d924..4949c363347 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/EnvelopeBuilder.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.geo.builders;
 
 import org.elasticsearch.common.geo.GeoShapeType;
+import org.elasticsearch.common.geo.parsers.GeoWKTParser;
 import org.elasticsearch.common.geo.parsers.ShapeParser;
 import org.locationtech.spatial4j.shape.Rectangle;
 import com.vividsolutions.jts.geom.Coordinate;
@@ -70,6 +71,28 @@ public class EnvelopeBuilder extends ShapeBuilder {
         return this.bottomRight;
     }
 
+    @Override
+    protected StringBuilder contentToWKT() {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append(GeoWKTParser.LPAREN);
+        // minX, maxX, maxY, minY
+        sb.append(topLeft.x);
+        sb.append(GeoWKTParser.COMMA);
+        sb.append(GeoWKTParser.SPACE);
+        sb.append(bottomRight.x);
+        sb.append(GeoWKTParser.COMMA);
+        sb.append(GeoWKTParser.SPACE);
+        // TODO support Z??
+        sb.append(topLeft.y);
+        sb.append(GeoWKTParser.COMMA);
+        sb.append(GeoWKTParser.SPACE);
+        sb.append(bottomRight.y);
+        sb.append(GeoWKTParser.RPAREN);
+
+        return sb;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java
index 3ea422265a7..84052939da4 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/GeometryCollectionBuilder.java
@@ -21,6 +21,7 @@ package org.elasticsearch.common.geo.builders;
 
 import org.elasticsearch.common.geo.GeoShapeType;
 import org.elasticsearch.common.geo.parsers.ShapeParser;
+import org.elasticsearch.common.geo.parsers.GeoWKTParser;
 import org.locationtech.spatial4j.shape.Shape;
 
 import org.elasticsearch.ElasticsearchException;
@@ -136,6 +137,23 @@ public class GeometryCollectionBuilder extends ShapeBuilder {
         return builder.endObject();
     }
 
+    @Override
+    protected StringBuilder contentToWKT() {
+        StringBuilder sb = new StringBuilder();
+        if (shapes.isEmpty()) {
+            sb.append(GeoWKTParser.EMPTY);
+        } else {
+            sb.append(GeoWKTParser.LPAREN);
+            sb.append(shapes.get(0).toWKT());
+            for (int i = 1; i < shapes.size(); ++i) {
+                sb.append(GeoWKTParser.COMMA);
+                sb.append(shapes.get(i).toWKT());
+            }
+            sb.append(GeoWKTParser.RPAREN);
+        }
+        return sb;
+    }
+
     @Override
     public GeoShapeType type() {
         return TYPE;
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java
index 1a4f71da2d4..34a8960f69c 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilder.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.common.geo.builders;
 
 import org.elasticsearch.common.geo.GeoShapeType;
+import org.elasticsearch.common.geo.parsers.GeoWKTParser;
 import org.elasticsearch.common.geo.parsers.ShapeParser;
-import org.locationtech.spatial4j.shape.Shape;
 import com.vividsolutions.jts.geom.Coordinate;
 import com.vividsolutions.jts.geom.Geometry;
 import com.vividsolutions.jts.geom.LineString;
@@ -82,6 +82,25 @@ public class MultiLineStringBuilder extends ShapeBuilder 0) {
+                sb.append(ShapeBuilder.coordinateListToWKT(lines.get(0).coordinates));
+            }
+            for (int i = 1; i < lines.size(); ++i) {
+                sb.append(GeoWKTParser.COMMA);
+                sb.append(ShapeBuilder.coordinateListToWKT(lines.get(i).coordinates));
+            }
+            sb.append(GeoWKTParser.RPAREN);
+        }
+        return sb;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java
index 3c002631b8d..aa577887e00 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/MultiPolygonBuilder.java
@@ -21,6 +21,7 @@ package org.elasticsearch.common.geo.builders;
 
 import org.elasticsearch.common.geo.GeoShapeType;
 import org.elasticsearch.common.geo.parsers.ShapeParser;
+import org.elasticsearch.common.geo.parsers.GeoWKTParser;
 import org.locationtech.spatial4j.shape.Shape;
 import com.vividsolutions.jts.geom.Coordinate;
 
@@ -101,6 +102,37 @@ public class MultiPolygonBuilder extends ShapeBuilder {
         return polygons;
     }
 
+    private static String polygonCoordinatesToWKT(PolygonBuilder polygon) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(GeoWKTParser.LPAREN);
+        sb.append(ShapeBuilder.coordinateListToWKT(polygon.shell().coordinates));
+        for (LineStringBuilder hole : polygon.holes()) {
+            sb.append(GeoWKTParser.COMMA);
+            sb.append(ShapeBuilder.coordinateListToWKT(hole.coordinates));
+        }
+        sb.append(GeoWKTParser.RPAREN);
+        return sb.toString();
+    }
+
+    @Override
+    protected StringBuilder contentToWKT() {
+        final StringBuilder sb = new StringBuilder();
+        if (polygons.isEmpty()) {
+            sb.append(GeoWKTParser.EMPTY);
+        } else {
+            sb.append(GeoWKTParser.LPAREN);
+            if (polygons.size() > 0) {
+                sb.append(polygonCoordinatesToWKT(polygons.get(0)));
+            }
+            for (int i = 1; i < polygons.size(); ++i) {
+                sb.append(GeoWKTParser.COMMA);
+                sb.append(polygonCoordinatesToWKT(polygons.get(i)));
+            }
+            sb.append(GeoWKTParser.RPAREN);
+        }
+        return sb;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
index 919aae37c73..ffcb44c9e46 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
@@ -729,6 +729,19 @@ public class PolygonBuilder extends ShapeBuilder {
         }
     }
 
+    @Override
+    protected StringBuilder contentToWKT() {
+        StringBuilder sb = new StringBuilder();
+        sb.append('(');
+        sb.append(ShapeBuilder.coordinateListToWKT(shell.coordinates));
+        for (LineStringBuilder hole : holes) {
+            sb.append(", ");
+            sb.append(ShapeBuilder.coordinateListToWKT(hole.coordinates));
+        }
+        sb.append(')');
+        return sb;
+    }
+
     @Override
     public int hashCode() {
         return Objects.hash(shell, holes, orientation);
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
index ef50a667faa..106c312a3bc 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
@@ -27,6 +27,7 @@ import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Assertions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.GeoShapeType;
+import org.elasticsearch.common.geo.parsers.GeoWKTParser;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -339,6 +340,47 @@ public abstract class ShapeBuilder>
         }
     }
 
+    protected StringBuilder contentToWKT() {
+        return coordinateListToWKT(this.coordinates);
+    }
+
+    public String toWKT() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(type().wktName());
+        sb.append(GeoWKTParser.SPACE);
+        sb.append(contentToWKT());
+        return sb.toString();
+    }
+
+    protected static StringBuilder coordinateListToWKT(final List coordinates) {
+        final StringBuilder sb = new StringBuilder();
+
+        if (coordinates.isEmpty()) {
+            sb.append(GeoWKTParser.EMPTY);
+        } else {
+            // walk through coordinates:
+            sb.append(GeoWKTParser.LPAREN);
+            sb.append(coordinateToWKT(coordinates.get(0)));
+            for (int i = 1; i < coordinates.size(); ++i) {
+                sb.append(GeoWKTParser.COMMA);
+                sb.append(GeoWKTParser.SPACE);
+                sb.append(coordinateToWKT(coordinates.get(i)));
+            }
+            sb.append(GeoWKTParser.RPAREN);
+        }
+
+        return sb;
+    }
+
+    private static String coordinateToWKT(final Coordinate coordinate) {
+        final StringBuilder sb = new StringBuilder();
+        sb.append(coordinate.x + GeoWKTParser.SPACE + coordinate.y);
+        if (Double.isNaN(coordinate.z) == false) {
+            sb.append(GeoWKTParser.SPACE + coordinate.z);
+        }
+        return sb.toString();
+    }
+
     protected static final IntersectionOrder INTERSECTION_ORDER = new IntersectionOrder();
 
     private static final class IntersectionOrder implements Comparator {
diff --git a/core/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java b/core/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java
new file mode 100644
index 00000000000..005caed53a7
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/geo/parsers/GeoWKTParser.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.geo.parsers;
+
+import com.vividsolutions.jts.geom.Coordinate;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.geo.GeoShapeType;
+
+import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
+import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
+import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
+import org.elasticsearch.common.geo.builders.LineStringBuilder;
+import org.elasticsearch.common.geo.builders.MultiLineStringBuilder;
+import org.elasticsearch.common.geo.builders.MultiPointBuilder;
+import org.elasticsearch.common.geo.builders.MultiPolygonBuilder;
+import org.elasticsearch.common.geo.builders.PointBuilder;
+import org.elasticsearch.common.geo.builders.PolygonBuilder;
+import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.io.FastStringReader;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.io.StreamTokenizer;
+import java.util.List;
+
+/**
+ * Parses shape geometry represented in WKT format
+ *
+ * complies with OGC® document: 12-063r5 and ISO/IEC 13249-3:2016 standard
+ * located at http://docs.opengeospatial.org/is/12-063r5/12-063r5.html
+ */
+public class GeoWKTParser {
+    public static final String EMPTY = "EMPTY";
+    public static final String SPACE = Loggers.SPACE;
+    public static final String LPAREN = "(";
+    public static final String RPAREN = ")";
+    public static final String COMMA = ",";
+    private static final String NAN = "NaN";
+
+    private static final String NUMBER = "";
+    private static final String EOF = "END-OF-STREAM";
+    private static final String EOL = "END-OF-LINE";
+
+    // no instance
+    private GeoWKTParser() {}
+
+    public static ShapeBuilder parse(XContentParser parser)
+            throws IOException, ElasticsearchParseException {
+        FastStringReader reader = new FastStringReader(parser.text());
+        try {
+            // setup the tokenizer; configured to read words w/o numbers
+            StreamTokenizer tokenizer = new StreamTokenizer(reader);
+            tokenizer.resetSyntax();
+            tokenizer.wordChars('a', 'z');
+            tokenizer.wordChars('A', 'Z');
+            tokenizer.wordChars(128 + 32, 255);
+            tokenizer.wordChars('0', '9');
+            tokenizer.wordChars('-', '-');
+            tokenizer.wordChars('+', '+');
+            tokenizer.wordChars('.', '.');
+            tokenizer.whitespaceChars(0, ' ');
+            tokenizer.commentChar('#');
+            ShapeBuilder builder = parseGeometry(tokenizer);
+            checkEOF(tokenizer);
+            return builder;
+        } finally {
+            reader.close();
+        }
+    }
+
+    /** parse geometry from the stream tokenizer */
+    private static ShapeBuilder parseGeometry(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        final GeoShapeType type = GeoShapeType.forName(nextWord(stream));
+        switch (type) {
+            case POINT:
+                return parsePoint(stream);
+            case MULTIPOINT:
+                return parseMultiPoint(stream);
+            case LINESTRING:
+                return parseLine(stream);
+            case MULTILINESTRING:
+                return parseMultiLine(stream);
+            case POLYGON:
+                return parsePolygon(stream);
+            case MULTIPOLYGON:
+                return parseMultiPolygon(stream);
+            case ENVELOPE:
+                return parseBBox(stream);
+            case GEOMETRYCOLLECTION:
+                return parseGeometryCollection(stream);
+            default:
+                throw new IllegalArgumentException("Unknown geometry type: " + type);
+        }
+    }
+
+    private static EnvelopeBuilder parseBBox(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+            return null;
+        }
+        double minLon = nextNumber(stream);
+        nextComma(stream);
+        double maxLon = nextNumber(stream);
+        nextComma(stream);
+        double maxLat = nextNumber(stream);
+        nextComma(stream);
+        double minLat = nextNumber(stream);
+        nextCloser(stream);
+        return new EnvelopeBuilder(new Coordinate(minLon, maxLat), new Coordinate(maxLon, minLat));
+    }
+
+    private static PointBuilder parsePoint(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+            return null;
+        }
+        PointBuilder pt = new PointBuilder(nextNumber(stream), nextNumber(stream));
+        if (isNumberNext(stream) == true) {
+            nextNumber(stream);
+        }
+        nextCloser(stream);
+        return pt;
+    }
+
+    private static List parseCoordinateList(StreamTokenizer stream)
+            throws IOException, ElasticsearchParseException {
+        CoordinatesBuilder coordinates = new CoordinatesBuilder();
+        boolean isOpenParen = false;
+        if (isNumberNext(stream) || (isOpenParen = nextWord(stream).equals(LPAREN))) {
+            coordinates.coordinate(parseCoordinate(stream));
+        }
+
+        if (isOpenParen && nextCloser(stream).equals(RPAREN) == false) {
+            throw new ElasticsearchParseException("expected: [{}]" + RPAREN + " but found: [{}]" + tokenString(stream), stream.lineno());
+        }
+
+        while (nextCloserOrComma(stream).equals(COMMA)) {
+            isOpenParen = false;
+            if (isNumberNext(stream) || (isOpenParen = nextWord(stream).equals(LPAREN))) {
+                coordinates.coordinate(parseCoordinate(stream));
+            }
+            if (isOpenParen && nextCloser(stream).equals(RPAREN) == false) {
+                throw new ElasticsearchParseException("expected: " + RPAREN + " but found: " + tokenString(stream), stream.lineno());
+            }
+        }
+        return coordinates.build();
+    }
+
+    private static Coordinate parseCoordinate(StreamTokenizer stream)
+            throws IOException, ElasticsearchParseException {
+        final double lon = nextNumber(stream);
+        final double lat = nextNumber(stream);
+        Double z = null;
+        if (isNumberNext(stream)) {
+            z = nextNumber(stream);
+        }
+        return z == null ? new Coordinate(lon, lat) : new Coordinate(lon, lat, z);
+    }
+
+    private static MultiPointBuilder parseMultiPoint(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        String token = nextEmptyOrOpen(stream);
+        if (token.equals(EMPTY)) {
+            return null;
+        }
+        return new MultiPointBuilder(parseCoordinateList(stream));
+    }
+
+    private static LineStringBuilder parseLine(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        String token = nextEmptyOrOpen(stream);
+        if (token.equals(EMPTY)) {
+            return null;
+        }
+        return new LineStringBuilder(parseCoordinateList(stream));
+    }
+
+    private static MultiLineStringBuilder parseMultiLine(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        String token = nextEmptyOrOpen(stream);
+        if (token.equals(EMPTY)) {
+            return null;
+        }
+        MultiLineStringBuilder builder = new MultiLineStringBuilder();
+        builder.linestring(parseLine(stream));
+        while (nextCloserOrComma(stream).equals(COMMA)) {
+            builder.linestring(parseLine(stream));
+        }
+        return builder;
+    }
+
+    private static PolygonBuilder parsePolygon(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+            return null;
+        }
+        PolygonBuilder builder = new PolygonBuilder(parseLine(stream), ShapeBuilder.Orientation.RIGHT);
+        while (nextCloserOrComma(stream).equals(COMMA)) {
+            builder.hole(parseLine(stream));
+        }
+        return builder;
+    }
+
+    private static MultiPolygonBuilder parseMultiPolygon(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+            return null;
+        }
+        MultiPolygonBuilder builder = new MultiPolygonBuilder().polygon(parsePolygon(stream));
+        while (nextCloserOrComma(stream).equals(COMMA)) {
+            builder.polygon(parsePolygon(stream));
+        }
+        return builder;
+    }
+
+    private static GeometryCollectionBuilder parseGeometryCollection(StreamTokenizer stream)
+            throws IOException, ElasticsearchParseException {
+        if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+            return null;
+        }
+        GeometryCollectionBuilder builder = new GeometryCollectionBuilder().shape(parseGeometry(stream));
+        while (nextCloserOrComma(stream).equals(COMMA)) {
+            builder.shape(parseGeometry(stream));
+        }
+        return builder;
+    }
+
+    /** next word in the stream */
+    private static String nextWord(StreamTokenizer stream) throws ElasticsearchParseException, IOException {
+        switch (stream.nextToken()) {
+            case StreamTokenizer.TT_WORD:
+                final String word = stream.sval;
+                return word.equalsIgnoreCase(EMPTY) ? EMPTY : word;
+            case '(': return LPAREN;
+            case ')': return RPAREN;
+            case ',': return COMMA;
+        }
+        throw new ElasticsearchParseException("expected word but found: " + tokenString(stream), stream.lineno());
+    }
+
+    private static double nextNumber(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        if (stream.nextToken() == StreamTokenizer.TT_WORD) {
+            if (stream.sval.equalsIgnoreCase(NAN)) {
+                return Double.NaN;
+            } else {
+                try {
+                    return Double.parseDouble(stream.sval);
+                } catch (NumberFormatException e) {
+                    throw new ElasticsearchParseException("invalid number found: " + stream.sval, stream.lineno());
+                }
+            }
+        }
+        throw new ElasticsearchParseException("expected number but found: " + tokenString(stream), stream.lineno());
+    }
+
+    private static String tokenString(StreamTokenizer stream) {
+        switch (stream.ttype) {
+            case StreamTokenizer.TT_WORD: return stream.sval;
+            case StreamTokenizer.TT_EOF: return EOF;
+            case StreamTokenizer.TT_EOL: return EOL;
+            case StreamTokenizer.TT_NUMBER: return NUMBER;
+        }
+        return "'" + (char) stream.ttype + "'";
+    }
+
+    private static boolean isNumberNext(StreamTokenizer stream) throws IOException {
+        final int type = stream.nextToken();
+        stream.pushBack();
+        return type == StreamTokenizer.TT_WORD;
+    }
+
+    private static String nextEmptyOrOpen(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        final String next = nextWord(stream);
+        if (next.equals(EMPTY) || next.equals(LPAREN)) {
+            return next;
+        }
+        throw new ElasticsearchParseException("expected " + EMPTY + " or " + LPAREN
+            + " but found: " + tokenString(stream), stream.lineno());
+    }
+
+    private static String nextCloser(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        if (nextWord(stream).equals(RPAREN)) {
+            return RPAREN;
+        }
+        throw new ElasticsearchParseException("expected " + RPAREN + " but found: " + tokenString(stream), stream.lineno());
+    }
+
+    private static String nextComma(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        if (nextWord(stream).equals(COMMA) == true) {
+            return COMMA;
+        }
+        throw new ElasticsearchParseException("expected " + COMMA + " but found: " + tokenString(stream), stream.lineno());
+    }
+
+    private static String nextCloserOrComma(StreamTokenizer stream) throws IOException, ElasticsearchParseException {
+        String token = nextWord(stream);
+        if (token.equals(COMMA) || token.equals(RPAREN)) {
+            return token;
+        }
+        throw new ElasticsearchParseException("expected " + COMMA + " or " + RPAREN
+            + " but found: " + tokenString(stream), stream.lineno());
+    }
+
+    /** next word in the stream */
+    private static void checkEOF(StreamTokenizer stream) throws ElasticsearchParseException, IOException {
+        if (stream.nextToken() != StreamTokenizer.TT_EOF) {
+            throw new ElasticsearchParseException("expected end of WKT string but found additional text: "
+                + tokenString(stream), stream.lineno());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java b/core/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java
index 39540f902fe..0ee3333c480 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java
@@ -51,6 +51,8 @@ public interface ShapeParser {
             return null;
         } if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
             return GeoJsonParser.parse(parser, shapeMapper);
+        } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
+            return GeoWKTParser.parse(parser);
         }
         throw new ElasticsearchParseException("shape must be an object consisting of type and coordinates");
     }
diff --git a/core/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java b/core/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java
index 3297e956e60..fff415de555 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java
@@ -18,10 +18,21 @@
  */
 package org.elasticsearch.common.geo;
 
+import com.vividsolutions.jts.geom.Geometry;
 import com.vividsolutions.jts.geom.GeometryFactory;
+import org.elasticsearch.common.geo.parsers.ShapeParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions;
+import org.locationtech.spatial4j.shape.Shape;
+import org.locationtech.spatial4j.shape.ShapeCollection;
+import org.locationtech.spatial4j.shape.jts.JtsGeometry;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 
 import static org.elasticsearch.common.geo.builders.ShapeBuilder.SPATIAL_CONTEXT;
 
@@ -35,4 +46,35 @@ abstract class BaseGeoParsingTestCase extends ESTestCase {
     public abstract void testParseMultiLineString() throws IOException;
     public abstract void testParsePolygon() throws IOException;
     public abstract void testParseMultiPolygon() throws IOException;
+    public abstract void testParseEnvelope() throws IOException;
+    public abstract void testParseGeometryCollection() throws IOException;
+
+    protected void assertValidException(XContentBuilder builder, Class expectedException) throws IOException {
+        XContentParser parser = createParser(builder);
+        parser.nextToken();
+        ElasticsearchGeoAssertions.assertValidException(parser, expectedException);
+    }
+
+    protected void assertGeometryEquals(Shape expected, XContentBuilder geoJson) throws IOException {
+        XContentParser parser = createParser(geoJson);
+        parser.nextToken();
+        ElasticsearchGeoAssertions.assertEquals(expected, ShapeParser.parse(parser).build());
+    }
+
+    protected ShapeCollection shapeCollection(Shape... shapes) {
+        return new ShapeCollection<>(Arrays.asList(shapes), SPATIAL_CONTEXT);
+    }
+
+    protected ShapeCollection shapeCollection(Geometry... geoms) {
+        List shapes = new ArrayList<>(geoms.length);
+        for (Geometry geom : geoms) {
+            shapes.add(jtsGeom(geom));
+        }
+        return new ShapeCollection<>(shapes, SPATIAL_CONTEXT);
+    }
+
+    protected JtsGeometry jtsGeom(Geometry geom) {
+        return new JtsGeometry(geom, SPATIAL_CONTEXT, false, false);
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java
index 32f384d96b1..fc987c7e3ca 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.geo;
 
 import com.vividsolutions.jts.geom.Coordinate;
-import com.vividsolutions.jts.geom.Geometry;
 import com.vividsolutions.jts.geom.LineString;
 import com.vividsolutions.jts.geom.LinearRing;
 import com.vividsolutions.jts.geom.MultiLineString;
@@ -39,12 +38,10 @@ import org.locationtech.spatial4j.shape.Circle;
 import org.locationtech.spatial4j.shape.Rectangle;
 import org.locationtech.spatial4j.shape.Shape;
 import org.locationtech.spatial4j.shape.ShapeCollection;
-import org.locationtech.spatial4j.shape.jts.JtsGeometry;
 import org.locationtech.spatial4j.shape.jts.JtsPoint;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
 import static org.elasticsearch.common.geo.builders.ShapeBuilder.SPATIAL_CONTEXT;
@@ -159,6 +156,7 @@ public class GeoJsonShapeParserTests extends BaseGeoParsingTestCase {
         assertGeometryEquals(jtsGeom(expectedLS), lineGeoJson);
     }
 
+    @Override
     public void testParseEnvelope() throws IOException {
         // test #1: envelope with expected coordinate order (TopLeft, BottomRight)
         XContentBuilder multilinesGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "envelope")
@@ -1033,27 +1031,4 @@ public class GeoJsonShapeParserTests extends BaseGeoParsingTestCase {
 
         ElasticsearchGeoAssertions.assertMultiPolygon(shape);
     }
-
-    private void assertGeometryEquals(Shape expected, XContentBuilder geoJson) throws IOException {
-        XContentParser parser = createParser(geoJson);
-        parser.nextToken();
-        ElasticsearchGeoAssertions.assertEquals(expected, ShapeParser.parse(parser).build());
-    }
-
-    private ShapeCollection shapeCollection(Shape... shapes) {
-        return new ShapeCollection<>(Arrays.asList(shapes), SPATIAL_CONTEXT);
-    }
-
-    private ShapeCollection shapeCollection(Geometry... geoms) {
-        List shapes = new ArrayList<>(geoms.length);
-        for (Geometry geom : geoms) {
-            shapes.add(jtsGeom(geom));
-        }
-        return new ShapeCollection<>(shapes, SPATIAL_CONTEXT);
-    }
-
-    private JtsGeometry jtsGeom(Geometry geom) {
-        return new JtsGeometry(geom, SPATIAL_CONTEXT, false, false);
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java
new file mode 100644
index 00000000000..191ce702052
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/geo/GeoWKTShapeParserTests.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.geo;
+
+import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.LineString;
+import com.vividsolutions.jts.geom.LinearRing;
+import com.vividsolutions.jts.geom.MultiLineString;
+import com.vividsolutions.jts.geom.Point;
+import com.vividsolutions.jts.geom.Polygon;
+import org.apache.lucene.geo.GeoTestUtil;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
+import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
+import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
+import org.elasticsearch.common.geo.builders.LineStringBuilder;
+import org.elasticsearch.common.geo.builders.MultiLineStringBuilder;
+import org.elasticsearch.common.geo.builders.MultiPointBuilder;
+import org.elasticsearch.common.geo.builders.MultiPolygonBuilder;
+import org.elasticsearch.common.geo.builders.PointBuilder;
+import org.elasticsearch.common.geo.builders.PolygonBuilder;
+import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.geo.parsers.GeoWKTParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.test.geo.RandomShapeGenerator;
+import org.locationtech.spatial4j.exception.InvalidShapeException;
+import org.locationtech.spatial4j.shape.Rectangle;
+import org.locationtech.spatial4j.shape.Shape;
+import org.locationtech.spatial4j.shape.ShapeCollection;
+import org.locationtech.spatial4j.shape.jts.JtsPoint;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.common.geo.builders.ShapeBuilder.SPATIAL_CONTEXT;
+
+/**
+ * Tests for {@code GeoWKTShapeParser}
+ */
+public class GeoWKTShapeParserTests extends BaseGeoParsingTestCase {
+
+    private static XContentBuilder toWKTContent(ShapeBuilder builder, boolean generateMalformed)
+            throws IOException {
+        String wkt = builder.toWKT();
+        if (generateMalformed) {
+            // malformed - extra paren
+            // TODO generate more malformed WKT
+            wkt += GeoWKTParser.RPAREN;
+        }
+        if (randomBoolean()) {
+            // test comments
+            wkt = "# " + wkt + "\n" + wkt;
+        }
+        return XContentFactory.jsonBuilder().value(wkt);
+    }
+
+    private void assertExpected(Shape expected, ShapeBuilder builder) throws IOException {
+        XContentBuilder xContentBuilder = toWKTContent(builder, false);
+        assertGeometryEquals(expected, xContentBuilder);
+    }
+
+    private void assertMalformed(Shape expected, ShapeBuilder builder) throws IOException {
+        XContentBuilder xContentBuilder = toWKTContent(builder, true);
+        assertValidException(xContentBuilder, ElasticsearchParseException.class);
+    }
+
+    @Override
+    public void testParsePoint() throws IOException {
+        GeoPoint p = RandomShapeGenerator.randomPoint(random());
+        Coordinate c = new Coordinate(p.lon(), p.lat());
+        Point expected = GEOMETRY_FACTORY.createPoint(c);
+        assertExpected(new JtsPoint(expected, SPATIAL_CONTEXT), new PointBuilder().coordinate(c));
+        assertMalformed(new JtsPoint(expected, SPATIAL_CONTEXT), new PointBuilder().coordinate(c));
+    }
+
+    @Override
+    public void testParseMultiPoint() throws IOException {
+        int numPoints = randomIntBetween(2, 100);
+        List coordinates = new ArrayList<>(numPoints);
+        Shape[] shapes = new Shape[numPoints];
+        GeoPoint p = new GeoPoint();
+        for (int i = 0; i < numPoints; ++i) {
+            p.reset(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude());
+            coordinates.add(new Coordinate(p.lon(), p.lat()));
+            shapes[i] = SPATIAL_CONTEXT.makePoint(p.lon(), p.lat());
+        }
+        ShapeCollection expected = shapeCollection(shapes);
+        assertExpected(expected, new MultiPointBuilder(coordinates));
+        assertMalformed(expected, new MultiPointBuilder(coordinates));
+    }
+
+    private List randomLineStringCoords() {
+        int numPoints = randomIntBetween(2, 100);
+        List coordinates = new ArrayList<>(numPoints);
+        GeoPoint p;
+        for (int i = 0; i < numPoints; ++i) {
+            p = RandomShapeGenerator.randomPointIn(random(), -90d, -90d, 90d, 90d);
+            coordinates.add(new Coordinate(p.lon(), p.lat()));
+        }
+        return coordinates;
+    }
+
+    @Override
+    public void testParseLineString() throws IOException {
+        List coordinates = randomLineStringCoords();
+        LineString expected = GEOMETRY_FACTORY.createLineString(coordinates.toArray(new Coordinate[coordinates.size()]));
+        assertExpected(jtsGeom(expected), new LineStringBuilder(coordinates));
+    }
+
+    @Override
+    public void testParseMultiLineString() throws IOException {
+        int numLineStrings = randomIntBetween(2, 8);
+        List lineStrings = new ArrayList<>(numLineStrings);
+        MultiLineStringBuilder builder = new MultiLineStringBuilder();
+        for (int j = 0; j < numLineStrings; ++j) {
+            List lsc = randomLineStringCoords();
+            Coordinate [] coords = lsc.toArray(new Coordinate[lsc.size()]);
+            lineStrings.add(GEOMETRY_FACTORY.createLineString(coords));
+            builder.linestring(new LineStringBuilder(lsc));
+        }
+        MultiLineString expected = GEOMETRY_FACTORY.createMultiLineString(
+            lineStrings.toArray(new LineString[lineStrings.size()]));
+        assertExpected(jtsGeom(expected), builder);
+        assertMalformed(jtsGeom(expected), builder);
+    }
+
+    @Override
+    public void testParsePolygon() throws IOException {
+        PolygonBuilder builder = PolygonBuilder.class.cast(
+            RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POLYGON));
+        Coordinate[] coords = builder.coordinates()[0][0];
+        LinearRing shell = GEOMETRY_FACTORY.createLinearRing(coords);
+        Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, null);
+        assertExpected(jtsGeom(expected), builder);
+        assertMalformed(jtsGeom(expected), builder);
+    }
+
+    @Override
+    public void testParseMultiPolygon() throws IOException {
+        int numPolys = randomIntBetween(2, 8);
+        MultiPolygonBuilder builder = new MultiPolygonBuilder();
+        PolygonBuilder pb;
+        Coordinate[] coordinates;
+        Polygon[] shapes = new Polygon[numPolys];
+        LinearRing shell;
+        for (int i = 0; i < numPolys; ++i) {
+            pb = PolygonBuilder.class.cast(RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POLYGON));
+            builder.polygon(pb);
+            coordinates = pb.coordinates()[0][0];
+            shell = GEOMETRY_FACTORY.createLinearRing(coordinates);
+            shapes[i] = GEOMETRY_FACTORY.createPolygon(shell, null);
+        }
+        Shape expected = shapeCollection(shapes);
+        assertExpected(expected, builder);
+        assertMalformed(expected, builder);
+    }
+
+    public void testParsePolygonWithHole() throws IOException {
+        // add 3d point to test ISSUE #10501
+        List shellCoordinates = new ArrayList<>();
+        shellCoordinates.add(new Coordinate(100, 0, 15.0));
+        shellCoordinates.add(new Coordinate(101, 0));
+        shellCoordinates.add(new Coordinate(101, 1));
+        shellCoordinates.add(new Coordinate(100, 1, 10.0));
+        shellCoordinates.add(new Coordinate(100, 0));
+
+        List holeCoordinates = new ArrayList<>();
+        holeCoordinates.add(new Coordinate(100.2, 0.2));
+        holeCoordinates.add(new Coordinate(100.8, 0.2));
+        holeCoordinates.add(new Coordinate(100.8, 0.8));
+        holeCoordinates.add(new Coordinate(100.2, 0.8));
+        holeCoordinates.add(new Coordinate(100.2, 0.2));
+
+        PolygonBuilder polygonWithHole = new PolygonBuilder(new CoordinatesBuilder().coordinates(shellCoordinates));
+        polygonWithHole.hole(new LineStringBuilder(holeCoordinates));
+
+        LinearRing shell = GEOMETRY_FACTORY.createLinearRing(
+            shellCoordinates.toArray(new Coordinate[shellCoordinates.size()]));
+        LinearRing[] holes = new LinearRing[1];
+        holes[0] = GEOMETRY_FACTORY.createLinearRing(
+            holeCoordinates.toArray(new Coordinate[holeCoordinates.size()]));
+        Polygon expected = GEOMETRY_FACTORY.createPolygon(shell, holes);
+
+        assertExpected(jtsGeom(expected), polygonWithHole);
+        assertMalformed(jtsGeom(expected), polygonWithHole);
+    }
+
+    public void testParseSelfCrossingPolygon() throws IOException {
+        // test self crossing ccw poly not crossing dateline
+        List shellCoordinates = new ArrayList<>();
+        shellCoordinates.add(new Coordinate(176, 15));
+        shellCoordinates.add(new Coordinate(-177, 10));
+        shellCoordinates.add(new Coordinate(-177, -10));
+        shellCoordinates.add(new Coordinate(176, -15));
+        shellCoordinates.add(new Coordinate(-177, 15));
+        shellCoordinates.add(new Coordinate(172, 0));
+        shellCoordinates.add(new Coordinate(176, 15));
+
+        PolygonBuilder poly = new PolygonBuilder(new CoordinatesBuilder().coordinates(shellCoordinates));
+        XContentBuilder builder = XContentFactory.jsonBuilder().value(poly.toWKT());
+        assertValidException(builder, InvalidShapeException.class);
+    }
+
+    public void testMalformedWKT() throws IOException {
+        // malformed points in a polygon is a common typo
+        String malformedWKT = "POLYGON ((100, 5) (100, 10) (90, 10), (90, 5), (100, 5)";
+        XContentBuilder builder = XContentFactory.jsonBuilder().value(malformedWKT);
+        assertValidException(builder, ElasticsearchParseException.class);
+    }
+
+    @Override
+    public void testParseEnvelope() throws IOException {
+        org.apache.lucene.geo.Rectangle r = GeoTestUtil.nextBox();
+        EnvelopeBuilder builder = new EnvelopeBuilder(new Coordinate(r.minLon, r.maxLat), new Coordinate(r.maxLon, r.minLat));
+        Rectangle expected = SPATIAL_CONTEXT.makeRectangle(r.minLon, r.maxLon, r.minLat, r.maxLat);
+        assertExpected(expected, builder);
+        assertMalformed(expected, builder);
+    }
+
+    public void testInvalidGeometryType() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().value("UnknownType (-1 -2)");
+        assertValidException(builder, IllegalArgumentException.class);
+    }
+
+    @Override
+    public void testParseGeometryCollection() throws IOException {
+        if (rarely()) {
+            // assert empty shape collection
+            GeometryCollectionBuilder builder = new GeometryCollectionBuilder();
+            Shape[] expected = new Shape[0];
+            assertEquals(shapeCollection(expected).isEmpty(), builder.build().isEmpty());
+        } else {
+            GeometryCollectionBuilder gcb = RandomShapeGenerator.createGeometryCollection(random());
+            assertExpected(gcb.build(), gcb);
+        }
+    }
+}
diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc
index b3420dbb58a..23caaf6a8ec 100644
--- a/docs/reference/mapping/types/geo-shape.asciidoc
+++ b/docs/reference/mapping/types/geo-shape.asciidoc
@@ -74,7 +74,7 @@ different ways. 1. Right-hand rule: `right`, `ccw`, `counterclockwise`,
 outer ring vertices in counterclockwise order with inner ring(s) vertices (holes)
 in clockwise order. Setting this parameter in the geo_shape mapping explicitly
 sets vertex order for the coordinate list of a geo_shape field but can be
-overridden in each individual GeoJSON document.
+overridden in each individual GeoJSON or WKT document.
 | `ccw`
 
 |`points_only` |Setting this option to `true` (defaults to `false`) configures
@@ -86,8 +86,9 @@ by improving point performance on a `geo_shape` field so that `geo_shape` querie
 optimal on a point only field.
 | `false`
 
-|`ignore_malformed` |If true, malformed geojson shapes are ignored. If false (default),
-malformed geojson shapes throw an exception and reject the whole document.
+|`ignore_malformed` |If true, malformed GeoJSON or WKT shapes are ignored. If
+false (default), malformed GeoJSON and WKT shapes throw an exception and reject the
+entire document.
 | `false`
 
 
@@ -204,29 +205,30 @@ overly bloating the resulting index too much relative to the input size.
 [float]
 ==== Input Structure
 
-The http://www.geojson.org[GeoJSON] format is used to represent
-http://geojson.org/geojson-spec.html#geometry-objects[shapes] as input
-as follows:
+Shapes can be represented using either the http://www.geojson.org[GeoJSON]
+or http://docs.opengeospatial.org/is/12-063r5/12-063r5.html[Well-Known Text]
+(WKT) format. The following table provides a mapping of GeoJSON and WKT
+to Elasticsearch types:
 
-[cols="<,<,<",options="header",]
+[cols="<,<,<,<",options="header",]
 |=======================================================================
-|GeoJSON Type |Elasticsearch Type |Description
+|GeoJSON Type |WKT Type |Elasticsearch Type |Description
 
-|`Point` |`point` |A single geographic coordinate.
-|`LineString` |`linestring` |An arbitrary line given two or more points.
-|`Polygon` |`polygon` |A _closed_ polygon whose first and last point
+|`Point` |`POINT` |`point` |A single geographic coordinate.
+|`LineString` |`LINESTRING` |`linestring` |An arbitrary line given two or more points.
+|`Polygon` |`POLYGON` |`polygon` |A _closed_ polygon whose first and last point
 must match, thus requiring `n + 1` vertices to create an `n`-sided
 polygon and a minimum of `4` vertices.
-|`MultiPoint` |`multipoint` |An array of unconnected, but likely related
+|`MultiPoint` |`MULTIPOINT` |`multipoint` |An array of unconnected, but likely related
 points.
-|`MultiLineString` |`multilinestring` |An array of separate linestrings.
-|`MultiPolygon` |`multipolygon` |An array of separate polygons.
-|`GeometryCollection` |`geometrycollection` | A GeoJSON shape similar to the
+|`MultiLineString` |`MULTILINESTRING` |`multilinestring` |An array of separate linestrings.
+|`MultiPolygon` |`MULTIPOLYGON` |`multipolygon` |An array of separate polygons.
+|`GeometryCollection` |`GEOMETRYCOLLECTION` |`geometrycollection` | A GeoJSON shape similar to the
 `multi*` shapes except that multiple types can coexist (e.g., a Point
 and a LineString).
-|`N/A` |`envelope` |A bounding rectangle, or envelope, specified by
+|`N/A` |`BBOX` |`envelope` |A bounding rectangle, or envelope, specified by
 specifying only the top left and bottom right points.
-|`N/A` |`circle` |A circle specified by a center point and radius with
+|`N/A` |`N/A` |`circle` |A circle specified by a center point and radius with
 units, which default to `METERS`.
 |=======================================================================
 
@@ -235,7 +237,7 @@ units, which default to `METERS`.
 For all types, both the inner `type` and `coordinates` fields are
 required.
 
-In GeoJSON, and therefore Elasticsearch, the correct *coordinate
+In GeoJSON and WKT, and therefore Elasticsearch, the correct *coordinate
 order is longitude, latitude (X, Y)* within coordinate arrays. This
 differs from many Geospatial APIs (e.g., Google Maps) that generally
 use the colloquial latitude, longitude (Y, X).
@@ -247,7 +249,7 @@ use the colloquial latitude, longitude (Y, X).
 
 A point is a single geographic coordinate, such as the location of a
 building or the current position given by a smartphone's Geolocation
-API.
+API. The following is an example of a point in GeoJSON.
 
 [source,js]
 --------------------------------------------------
@@ -261,12 +263,24 @@ POST /example/doc
 --------------------------------------------------
 // CONSOLE
 
+The following is an example of a point in WKT:
+
+[source,js]
+--------------------------------------------------
+POST /example/doc
+{
+    "location" : "POINT (-77.03653 38.897676)"
+}
+--------------------------------------------------
+// CONSOLE
+
 [float]
 ===== http://geojson.org/geojson-spec.html#id3[LineString]
 
 A `linestring` defined by an array of two or more positions. By
 specifying only two points, the `linestring` will represent a straight
-line.  Specifying more than two points creates an arbitrary path.
+line.  Specifying more than two points creates an arbitrary path. The
+following is an example of a LineString in GeoJSON.
 
 [source,js]
 --------------------------------------------------
@@ -280,6 +294,17 @@ POST /example/doc
 --------------------------------------------------
 // CONSOLE
 
+The following is an example of a LineString in WKT:
+
+[source,js]
+--------------------------------------------------
+POST /example/doc
+{
+    "location" : "LINESTRING (-77.03653 38.897676, -77.009051 38.889939)"
+}
+--------------------------------------------------
+// CONSOLE
+
 The above `linestring` would draw a straight line starting at the White
 House to the US Capitol Building.
 
@@ -288,7 +313,7 @@ House to the US Capitol Building.
 
 A polygon is defined by a list of a list of points. The first and last
 points in each (outer) list must be the same (the polygon must be
-closed).
+closed). The following is an example of a Polygon in GeoJSON.
 
 [source,js]
 --------------------------------------------------
@@ -304,8 +329,20 @@ POST /example/doc
 --------------------------------------------------
 // CONSOLE
 
+The following is an example of a Polygon in WKT:
+
+[source,js]
+--------------------------------------------------
+POST /example/doc
+{
+    "location" : "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0))"
+}
+--------------------------------------------------
+// CONSOLE
+
 The first array represents the outer boundary of the polygon, the other
-arrays represent the interior shapes ("holes"):
+arrays represent the interior shapes ("holes"). The following is a GeoJSON example
+of a polygon with a hole:
 
 [source,js]
 --------------------------------------------------
@@ -323,9 +360,21 @@ POST /example/doc
 // CONSOLE
 // TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
-*IMPORTANT NOTE:* GeoJSON does not mandate a specific order for vertices thus ambiguous
-polygons around the dateline and poles are possible. To alleviate ambiguity
-the Open Geospatial Consortium (OGC)
+The following is an example of a Polygon with a hole in WKT:
+
+[source,js]
+--------------------------------------------------
+POST /example/doc
+{
+    "location" : "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0), (100.2 0.2, 100.8 0.2, 100.8 0.8, 100.2 0.8, 100.2 0.2))"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
+
+*IMPORTANT NOTE:* GeoJSON and WKT do not enforce a specific order for vertices
+thus ambiguous polygons around the dateline and poles are possible. To alleviate
+ambiguity the Open Geospatial Consortium (OGC)
 http://www.opengeospatial.org/standards/sfa[Simple Feature Access] specification
 defines the following vertex ordering:
 
@@ -380,7 +429,7 @@ POST /example/doc
 [float]
 ===== http://www.geojson.org/geojson-spec.html#id5[MultiPoint]
 
-A list of geojson points.
+The following is an example of a list of geojson points:
 
 [source,js]
 --------------------------------------------------
@@ -396,10 +445,21 @@ POST /example/doc
 --------------------------------------------------
 // CONSOLE
 
+The following is an example of a list of WKT points:
+
+[source,js]
+--------------------------------------------------
+POST /example/doc
+{
+    "location" : "MULTIPOINT (102.0 2.0, 103.0 2.0)"
+}
+--------------------------------------------------
+// CONSOLE
+
 [float]
 ===== http://www.geojson.org/geojson-spec.html#id6[MultiLineString]
 
-A list of geojson linestrings.
+The following is an example of a list of geojson linestrings:
 
 [source,js]
 --------------------------------------------------
@@ -418,10 +478,22 @@ POST /example/doc
 // CONSOLE
 // TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
+The following is an example of a list of WKT linestrings:
+
+[source,js]
+--------------------------------------------------
+POST /example/doc
+{
+    "location" : "MULTILINESTRING ((102.0 2.0, 103.0 2.0, 103.0 3.0, 102.0 3.0), (100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0), (100.2 0.2, 100.8 0.2, 100.8 0.8, 100.2 0.8))"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
+
 [float]
 ===== http://www.geojson.org/geojson-spec.html#id7[MultiPolygon]
 
-A list of geojson polygons.
+The following is an example of a list of geojson polygons (second polygon contains a hole):
 
 [source,js]
 --------------------------------------------------
@@ -440,10 +512,22 @@ POST /example/doc
 // CONSOLE
 // TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
+The following is an example of a list of WKT polygons (second polygon contains a hole):
+
+[source,js]
+--------------------------------------------------
+POST /example/doc
+{
+    "location" : "MULTIPOLYGON (((102.0 2.0, 103.0 2.0, 103.0 3.0, 102.0 3.0, 102.0 2.0)), ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0), (100.2 0.2, 100.8 0.2, 100.8 0.8, 100.2 0.8, 100.2 0.2)))"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
+
 [float]
 ===== http://geojson.org/geojson-spec.html#geometrycollection[Geometry Collection]
 
-A collection of geojson geometry objects.
+The following is an example of a collection of geojson geometry objects:
 
 [source,js]
 --------------------------------------------------
@@ -467,6 +551,19 @@ POST /example/doc
 // CONSOLE
 // TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
+The following is an example of a collection of WKT geometry objects:
+
+[source,js]
+--------------------------------------------------
+POST /example/doc
+{
+    "location" : "GEOMETRYCOLLECTION (POINT (100.0 0.0), LINESTRING (101.0 0.0, 102.0 1.0))"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
+
+
 [float]
 ===== Envelope
 
@@ -487,6 +584,20 @@ POST /example/doc
 // CONSOLE
 // TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
 
+The following is an example of an envelope using the WKT BBOX format:
+
+*NOTE:* WKT specification expects the following order: minLon, maxLon, maxLat, minLat.
+
+[source,js]
+--------------------------------------------------
+POST /example/doc
+{
+    "location" : "BBOX (-45.0, 45.0, 45.0, -45.0)"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/23836]
+
 [float]
 ===== Circle
 
@@ -509,6 +620,8 @@ POST /example/doc
 Note: The inner `radius` field is required. If not specified, then
 the units of the `radius` will default to `METERS`.
 
+*NOTE:* Neither GeoJSON or WKT support a point-radius circle type.
+
 [float]
 ==== Sorting and Retrieving index Shapes
 

From 25ec068aef928a355805f8d2f70fad423b473a76 Mon Sep 17 00:00:00 2001
From: debadair 
Date: Tue, 5 Dec 2017 10:58:52 -0800
Subject: [PATCH 187/297] [DOCS] Added link to upgrade guide and bumped the
 upgrade topic up to the top level (#27621)

* [DOCS] Added link to the upgrade guide & tweaked the intro.

* [DOCS] Bumped upgrade topic up to the top level of the TOC
---
 docs/reference/index-shared1.asciidoc         |  2 ++
 docs/reference/setup.asciidoc                 |  2 --
 docs/reference/{setup => }/upgrade.asciidoc   | 22 ++++++++++++++-----
 .../upgrade/cluster_restart.asciidoc          |  2 +-
 .../upgrade/disable-shard-alloc.asciidoc      |  0
 .../upgrade/reindex_upgrade.asciidoc          |  6 ++---
 .../upgrade/rolling_upgrade.asciidoc          |  2 +-
 .../upgrade/set-paths-tip.asciidoc            |  0
 .../upgrade/shut-down-node.asciidoc           |  0
 .../{setup => }/upgrade/synced-flush.asciidoc |  0
 .../{setup => }/upgrade/upgrade-node.asciidoc |  0
 11 files changed, 23 insertions(+), 13 deletions(-)
 rename docs/reference/{setup => }/upgrade.asciidoc (87%)
 rename docs/reference/{setup => }/upgrade/cluster_restart.asciidoc (99%)
 rename docs/reference/{setup => }/upgrade/disable-shard-alloc.asciidoc (100%)
 rename docs/reference/{setup => }/upgrade/reindex_upgrade.asciidoc (98%)
 rename docs/reference/{setup => }/upgrade/rolling_upgrade.asciidoc (99%)
 rename docs/reference/{setup => }/upgrade/set-paths-tip.asciidoc (100%)
 rename docs/reference/{setup => }/upgrade/shut-down-node.asciidoc (100%)
 rename docs/reference/{setup => }/upgrade/synced-flush.asciidoc (100%)
 rename docs/reference/{setup => }/upgrade/upgrade-node.asciidoc (100%)

diff --git a/docs/reference/index-shared1.asciidoc b/docs/reference/index-shared1.asciidoc
index 9325bd6e73e..ae208e29011 100644
--- a/docs/reference/index-shared1.asciidoc
+++ b/docs/reference/index-shared1.asciidoc
@@ -2,3 +2,5 @@
 include::getting-started.asciidoc[]
 
 include::setup.asciidoc[]
+
+include::upgrade.asciidoc[]
diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc
index 34ac8c0bd66..0dcafe375cf 100644
--- a/docs/reference/setup.asciidoc
+++ b/docs/reference/setup.asciidoc
@@ -50,6 +50,4 @@ include::setup/sysconfig.asciidoc[]
 
 include::setup/bootstrap-checks.asciidoc[]
 
-include::setup/upgrade.asciidoc[]
-
 include::setup/stopping.asciidoc[]
diff --git a/docs/reference/setup/upgrade.asciidoc b/docs/reference/upgrade.asciidoc
similarity index 87%
rename from docs/reference/setup/upgrade.asciidoc
rename to docs/reference/upgrade.asciidoc
index 172ae13e32d..5218856d0c9 100644
--- a/docs/reference/setup/upgrade.asciidoc
+++ b/docs/reference/upgrade.asciidoc
@@ -1,5 +1,19 @@
 [[setup-upgrade]]
-== Upgrading Elasticsearch
+= Upgrade Elasticsearch
+
+[partintro]
+--
+Elasticsearch can usually be upgraded using a <>
+process so upgrading does not interrupt service. However, you might
+need to <> indices created in older versions.
+Upgrades across major versions prior to 6.0 require a <>.
+
+When upgrading to a new version of Elasticsearch, you need to upgrade each
+of the products in your Elastic Stack. The steps you need to take to upgrade
+differ depending on which products you are using. Want a list that's tailored
+to your stack? Try out our {upgrade_guide}[Interactive Upgrade Guide]. For
+more information about upgrading your stack, see {stack-ref}[Upgrading the
+Elastic Stack].
 
 [IMPORTANT]
 ===========================================
@@ -17,10 +31,6 @@ your data.
 
 ===========================================
 
-Elasticsearch can usually be upgraded using a <>
-process so upgrading does not interrupt service. However, you might
-need to <> indices created in older versions.
-Upgrades across major versions prior to 6.0 require a <>.
 
 The following table shows when you can perform a rolling upgrade, when you
 need to reindex or delete old indices, and when a full cluster restart is
@@ -56,7 +66,7 @@ For information about how to upgrade old indices, see <>.
 
 ===============================================
-
+--
 
 include::upgrade/rolling_upgrade.asciidoc[]
 
diff --git a/docs/reference/setup/upgrade/cluster_restart.asciidoc b/docs/reference/upgrade/cluster_restart.asciidoc
similarity index 99%
rename from docs/reference/setup/upgrade/cluster_restart.asciidoc
rename to docs/reference/upgrade/cluster_restart.asciidoc
index edb23cf7e6d..7989f14eaad 100644
--- a/docs/reference/setup/upgrade/cluster_restart.asciidoc
+++ b/docs/reference/upgrade/cluster_restart.asciidoc
@@ -1,5 +1,5 @@
 [[restart-upgrade]]
-=== Full cluster restart upgrade
+== Full cluster restart upgrade
 
 A full cluster restart upgrade requires that you shut all nodes in the cluster
 down, upgrade them, and restart the cluster. A full cluster restart was required
diff --git a/docs/reference/setup/upgrade/disable-shard-alloc.asciidoc b/docs/reference/upgrade/disable-shard-alloc.asciidoc
similarity index 100%
rename from docs/reference/setup/upgrade/disable-shard-alloc.asciidoc
rename to docs/reference/upgrade/disable-shard-alloc.asciidoc
diff --git a/docs/reference/setup/upgrade/reindex_upgrade.asciidoc b/docs/reference/upgrade/reindex_upgrade.asciidoc
similarity index 98%
rename from docs/reference/setup/upgrade/reindex_upgrade.asciidoc
rename to docs/reference/upgrade/reindex_upgrade.asciidoc
index 04ef0f18b60..dc3c72fee22 100644
--- a/docs/reference/setup/upgrade/reindex_upgrade.asciidoc
+++ b/docs/reference/upgrade/reindex_upgrade.asciidoc
@@ -1,5 +1,5 @@
 [[reindex-upgrade]]
-=== Reindex before upgrading
+== Reindex before upgrading
 
 Elasticsearch can read indices created in the *previous major version*.
 Older indices must be reindexed or deleted. Elasticsearch 6.x
@@ -56,7 +56,7 @@ been deleted.
 
 
 [[reindex-upgrade-inplace]]
-==== Reindex in place
+=== Reindex in place
 
 To manually reindex your old indices with the <>:
 
@@ -88,7 +88,7 @@ regular part of your Elastic Stack. For more information, see
 *******************************************
 
 [[reindex-upgrade-remote]]
-==== Reindex from a remote cluster
+=== Reindex from a remote cluster
 
 You can use <> to migrate indices from
 your old cluster to a new 6.x cluster. This enables you move to 6.x from a
diff --git a/docs/reference/setup/upgrade/rolling_upgrade.asciidoc b/docs/reference/upgrade/rolling_upgrade.asciidoc
similarity index 99%
rename from docs/reference/setup/upgrade/rolling_upgrade.asciidoc
rename to docs/reference/upgrade/rolling_upgrade.asciidoc
index 09427ba26ad..f52f354b80b 100644
--- a/docs/reference/setup/upgrade/rolling_upgrade.asciidoc
+++ b/docs/reference/upgrade/rolling_upgrade.asciidoc
@@ -1,5 +1,5 @@
 [[rolling-upgrades]]
-=== Rolling upgrades
+== Rolling upgrades
 
 A rolling upgrade allows an Elasticsearch cluster to be upgraded one node at
 a time so upgrading does not interrupt service. Running multiple versions of
diff --git a/docs/reference/setup/upgrade/set-paths-tip.asciidoc b/docs/reference/upgrade/set-paths-tip.asciidoc
similarity index 100%
rename from docs/reference/setup/upgrade/set-paths-tip.asciidoc
rename to docs/reference/upgrade/set-paths-tip.asciidoc
diff --git a/docs/reference/setup/upgrade/shut-down-node.asciidoc b/docs/reference/upgrade/shut-down-node.asciidoc
similarity index 100%
rename from docs/reference/setup/upgrade/shut-down-node.asciidoc
rename to docs/reference/upgrade/shut-down-node.asciidoc
diff --git a/docs/reference/setup/upgrade/synced-flush.asciidoc b/docs/reference/upgrade/synced-flush.asciidoc
similarity index 100%
rename from docs/reference/setup/upgrade/synced-flush.asciidoc
rename to docs/reference/upgrade/synced-flush.asciidoc
diff --git a/docs/reference/setup/upgrade/upgrade-node.asciidoc b/docs/reference/upgrade/upgrade-node.asciidoc
similarity index 100%
rename from docs/reference/setup/upgrade/upgrade-node.asciidoc
rename to docs/reference/upgrade/upgrade-node.asciidoc

From f4fb4d3bf5cecd25fc32bc96c7fa83125a0b0486 Mon Sep 17 00:00:00 2001
From: Luca Cavanna 
Date: Tue, 5 Dec 2017 20:31:29 +0100
Subject: [PATCH 188/297] Add support for filtering mappings fields (#27603)

Add support for filtering fields returned as part of mappings in get index, get mappings, get field mappings and field capabilities API.

Plugins can plug in their own function, which receives the index as argument, and return a predicate which controls whether each field is included or not in the returned output.
---
 .../indices/get/TransportGetIndexAction.java  |  24 +-
 .../TransportGetFieldMappingsIndexAction.java |  58 +--
 .../get/TransportGetMappingsAction.java       |  29 +-
 ...TransportFieldCapabilitiesIndexAction.java |   6 +-
 .../cluster/metadata/MappingMetaData.java     |  17 -
 .../cluster/metadata/MetaData.java            | 115 +++++-
 .../common/util/concurrent/ThreadContext.java |   2 -
 .../index/mapper/MapperService.java           |   2 +
 .../elasticsearch/indices/IndicesModule.java  |  99 ++++-
 .../elasticsearch/indices/IndicesService.java |  20 +
 .../indices/mapper/MapperRegistry.java        |  26 +-
 .../elasticsearch/plugins/MapperPlugin.java   |  33 +-
 .../MetaDataIndexUpgradeServiceTests.java     |  47 ++-
 .../cluster/metadata/MetaDataTests.java       | 370 ++++++++++++++++++
 .../elasticsearch/index/codec/CodecTests.java |   3 +-
 .../mapper/ExternalFieldMapperTests.java      |   8 +-
 .../mapper/FieldFilterMapperPluginTests.java  | 326 +++++++++++++++
 .../indices/IndicesModuleTests.java           | 161 +++++++-
 .../indices/IndicesServiceTests.java          |   9 +-
 .../murmur3/Murmur3FieldMapperTests.java      |   3 +-
 .../index/mapper/MockFieldFilterPlugin.java   |  35 ++
 .../elasticsearch/test/ESIntegTestCase.java   |   4 +
 22 files changed, 1253 insertions(+), 144 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldFilterPlugin.java

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
index c9cf3257c76..b383c02be74 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java
@@ -36,9 +36,11 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.List;
 
 /**
@@ -46,10 +48,15 @@ import java.util.List;
  */
 public class TransportGetIndexAction extends TransportClusterInfoAction {
 
+    private final IndicesService indicesService;
+
     @Inject
     public TransportGetIndexAction(Settings settings, TransportService transportService, ClusterService clusterService,
-                                   ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, GetIndexAction.NAME, transportService, clusterService, threadPool, actionFilters, GetIndexRequest::new, indexNameExpressionResolver);
+                                   ThreadPool threadPool, ActionFilters actionFilters,
+                                   IndexNameExpressionResolver indexNameExpressionResolver, IndicesService indicesService) {
+        super(settings, GetIndexAction.NAME, transportService, clusterService, threadPool, actionFilters, GetIndexRequest::new,
+                indexNameExpressionResolver);
+        this.indicesService = indicesService;
     }
 
     @Override
@@ -60,7 +67,8 @@ public class TransportGetIndexAction extends TransportClusterInfoAction metadataFieldPredicate = indicesService::isMetaDataField;
+        Predicate fieldPredicate = metadataFieldPredicate.or(indicesService.getFieldFilter().apply(shardId.getIndexName()));
+
         Collection typeIntersection;
         if (request.types().length == 0) {
             typeIntersection = indexService.mapperService().types();
@@ -104,16 +110,15 @@ public class TransportGetFieldMappingsIndexAction extends TransportSingleShardAc
             }
         }
 
-        MapBuilder> typeMappings = new MapBuilder<>();
+        Map> typeMappings = new HashMap<>();
         for (String type : typeIntersection) {
             DocumentMapper documentMapper = indexService.mapperService().documentMapper(type);
-            Map fieldMapping = findFieldMappingsByType(documentMapper, request);
+            Map fieldMapping = findFieldMappingsByType(fieldPredicate, documentMapper, request);
             if (!fieldMapping.isEmpty()) {
                 typeMappings.put(type, fieldMapping);
             }
         }
-
-        return new GetFieldMappingsResponse(singletonMap(shardId.getIndexName(), typeMappings.immutableMap()));
+        return new GetFieldMappingsResponse(singletonMap(shardId.getIndexName(), Collections.unmodifiableMap(typeMappings)));
     }
 
     @Override
@@ -163,47 +168,50 @@ public class TransportGetFieldMappingsIndexAction extends TransportSingleShardAc
         }
     };
 
-    private Map findFieldMappingsByType(DocumentMapper documentMapper, GetFieldMappingsIndexRequest request) {
-        MapBuilder fieldMappings = new MapBuilder<>();
+    private static Map findFieldMappingsByType(Predicate fieldPredicate,
+                                                                             DocumentMapper documentMapper,
+                                                                             GetFieldMappingsIndexRequest request) {
+        Map fieldMappings = new HashMap<>();
         final DocumentFieldMappers allFieldMappers = documentMapper.mappers();
         for (String field : request.fields()) {
             if (Regex.isMatchAllPattern(field)) {
                 for (FieldMapper fieldMapper : allFieldMappers) {
-                    addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings, request.includeDefaults());
+                    addFieldMapper(fieldPredicate, fieldMapper.fieldType().name(), fieldMapper, fieldMappings, request.includeDefaults());
                 }
             } else if (Regex.isSimpleMatchPattern(field)) {
                 for (FieldMapper fieldMapper : allFieldMappers) {
                     if (Regex.simpleMatch(field, fieldMapper.fieldType().name())) {
-                        addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings,
-                            request.includeDefaults());
+                        addFieldMapper(fieldPredicate,  fieldMapper.fieldType().name(),
+                                fieldMapper, fieldMappings, request.includeDefaults());
                     }
                 }
             } else {
                 // not a pattern
                 FieldMapper fieldMapper = allFieldMappers.smartNameFieldMapper(field);
                 if (fieldMapper != null) {
-                    addFieldMapper(field, fieldMapper, fieldMappings, request.includeDefaults());
+                    addFieldMapper(fieldPredicate, field, fieldMapper, fieldMappings, request.includeDefaults());
                 } else if (request.probablySingleFieldRequest()) {
                     fieldMappings.put(field, FieldMappingMetaData.NULL);
                 }
             }
         }
-        return fieldMappings.immutableMap();
+        return Collections.unmodifiableMap(fieldMappings);
     }
 
-    private void addFieldMapper(String field, FieldMapper fieldMapper, MapBuilder fieldMappings, boolean includeDefaults) {
+    private static void addFieldMapper(Predicate fieldPredicate,
+                                       String field, FieldMapper fieldMapper, Map fieldMappings,
+                                       boolean includeDefaults) {
         if (fieldMappings.containsKey(field)) {
             return;
         }
-        try {
-            XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-            builder.startObject();
-            fieldMapper.toXContent(builder, includeDefaults ? includeDefaultsParams : ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-            fieldMappings.put(field, new FieldMappingMetaData(fieldMapper.fieldType().name(), builder.bytes()));
-        } catch (IOException e) {
-            throw new ElasticsearchException("failed to serialize XContent of field [" + field + "]", e);
+        if (fieldPredicate.test(field)) {
+            try {
+                BytesReference bytes = XContentHelper.toXContent(fieldMapper, XContentType.JSON,
+                        includeDefaults ? includeDefaultsParams : ToXContent.EMPTY_PARAMS, false);
+                fieldMappings.put(field, new FieldMappingMetaData(fieldMapper.fieldType().name(), bytes));
+            } catch (IOException e) {
+                throw new ElasticsearchException("failed to serialize XContent of field [" + field + "]", e);
+            }
         }
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java
index 3189a5a15c2..8ad2ce5475f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java
@@ -31,15 +31,23 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 public class TransportGetMappingsAction extends TransportClusterInfoAction {
 
+    private final IndicesService indicesService;
+
     @Inject
     public TransportGetMappingsAction(Settings settings, TransportService transportService, ClusterService clusterService,
-                                      ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, GetMappingsAction.NAME, transportService, clusterService, threadPool, actionFilters, GetMappingsRequest::new, indexNameExpressionResolver);
+                                      ThreadPool threadPool, ActionFilters actionFilters,
+                                      IndexNameExpressionResolver indexNameExpressionResolver, IndicesService indicesService) {
+        super(settings, GetMappingsAction.NAME, transportService, clusterService, threadPool, actionFilters, GetMappingsRequest::new,
+                indexNameExpressionResolver);
+        this.indicesService = indicesService;
     }
 
     @Override
@@ -50,7 +58,8 @@ public class TransportGetMappingsAction extends TransportClusterInfoAction listener) {
+    protected void doMasterOperation(final GetMappingsRequest request, String[] concreteIndices, final ClusterState state,
+                                     final ActionListener listener) {
         logger.trace("serving getMapping request based on version {}", state.version());
-        ImmutableOpenMap> result = state.metaData().findMappings(
-                concreteIndices, request.types()
-        );
-        listener.onResponse(new GetMappingsResponse(result));
+        try {
+            ImmutableOpenMap> result =
+                    state.metaData().findMappings(concreteIndices, request.types(), indicesService.getFieldFilter());
+            listener.onResponse(new GetMappingsResponse(result));
+        } catch (IOException e) {
+            listener.onFailure(e);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
index b9e6f56b6d7..b24dc685df6 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
@@ -40,6 +40,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Predicate;
 
 public class TransportFieldCapabilitiesIndexAction extends TransportSingleShardAction {
@@ -77,12 +78,15 @@ public class TransportFieldCapabilitiesIndexAction extends TransportSingleShardA
         for (String field : request.fields()) {
             fieldNames.addAll(mapperService.simpleMatchToIndexNames(field));
         }
+        Predicate fieldPredicate = indicesService.getFieldFilter().apply(shardId.getIndexName());
         Map responseMap = new HashMap<>();
         for (String field : fieldNames) {
             MappedFieldType ft = mapperService.fullName(field);
             if (ft != null) {
                 FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable());
-                responseMap.put(field, fieldCap);
+                if (indicesService.isMetaDataField(field) || fieldPredicate.test(field)) {
+                    responseMap.put(field, fieldCap);
+                }
             }
         }
         return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), responseMap);
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
index 83a06d9c4ca..9cbfb2ec71f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
@@ -107,15 +107,6 @@ public class MappingMetaData extends AbstractDiffable {
         initMappers(withoutType);
     }
 
-    private MappingMetaData() {
-        this.type = "";
-        try {
-            this.source = new CompressedXContent("{}");
-        } catch (IOException ex) {
-            throw new IllegalStateException("Cannot create MappingMetaData prototype", ex);
-        }
-    }
-
     private void initMappers(Map withoutType) {
         if (withoutType.containsKey("_routing")) {
             boolean required = false;
@@ -143,13 +134,6 @@ public class MappingMetaData extends AbstractDiffable {
         }
     }
 
-    public MappingMetaData(String type, CompressedXContent source, Routing routing, boolean hasParentField) {
-        this.type = type;
-        this.source = source;
-        this.routing = routing;
-        this.hasParentField = hasParentField;
-    }
-
     void updateDefaultMapping(MappingMetaData defaultMapping) {
         if (routing == Routing.EMPTY) {
             routing = defaultMapping.routing();
@@ -250,5 +234,4 @@ public class MappingMetaData extends AbstractDiffable {
     public static Diff readDiffFrom(StreamInput in) throws IOException {
         return readDiffFrom(MappingMetaData::new, in);
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index c582f372e51..0c543d5ffe3 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -53,6 +53,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.MetaDataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -69,6 +70,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.function.Function;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
@@ -324,32 +327,38 @@ public class MetaData implements Iterable, Diffable, To
         return false;
     }
 
-    /*
-     * Finds all mappings for types and concrete indices. Types are expanded to
-     * include all types that match the glob patterns in the types array. Empty
-     * types array, null or {"_all"} will be expanded to all types available for
-     * the given indices.
+    /**
+     * Finds all mappings for types and concrete indices. Types are expanded to include all types that match the glob
+     * patterns in the types array. Empty types array, null or {"_all"} will be expanded to all types available for
+     * the given indices. Only fields that match the provided field filter will be returned (default is a predicate
+     * that always returns true, which can be overridden via plugins)
+     *
+     * @see MapperPlugin#getFieldFilter()
+     *
      */
-    public ImmutableOpenMap> findMappings(String[] concreteIndices, final String[] types) {
+    public ImmutableOpenMap> findMappings(String[] concreteIndices,
+                                                                                            final String[] types,
+                                                                                            Function> fieldFilter)
+            throws IOException {
         assert types != null;
         assert concreteIndices != null;
         if (concreteIndices.length == 0) {
             return ImmutableOpenMap.of();
         }
 
+        boolean isAllTypes = isAllTypes(types);
         ImmutableOpenMap.Builder> indexMapBuilder = ImmutableOpenMap.builder();
         Iterable intersection = HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
         for (String index : intersection) {
             IndexMetaData indexMetaData = indices.get(index);
-            ImmutableOpenMap.Builder filteredMappings;
-            if (isAllTypes(types)) {
-                indexMapBuilder.put(index, indexMetaData.getMappings()); // No types specified means get it all
-
+            Predicate fieldPredicate = fieldFilter.apply(index);
+            if (isAllTypes) {
+                indexMapBuilder.put(index, filterFields(indexMetaData.getMappings(), fieldPredicate));
             } else {
-                filteredMappings = ImmutableOpenMap.builder();
+                ImmutableOpenMap.Builder filteredMappings = ImmutableOpenMap.builder();
                 for (ObjectObjectCursor cursor : indexMetaData.getMappings()) {
                     if (Regex.simpleMatch(types, cursor.key)) {
-                        filteredMappings.put(cursor.key, cursor.value);
+                        filteredMappings.put(cursor.key, filterFields(cursor.value, fieldPredicate));
                     }
                 }
                 if (!filteredMappings.isEmpty()) {
@@ -360,6 +369,88 @@ public class MetaData implements Iterable, Diffable, To
         return indexMapBuilder.build();
     }
 
+    private static ImmutableOpenMap filterFields(ImmutableOpenMap mappings,
+                                                                          Predicate fieldPredicate) throws IOException {
+        if (fieldPredicate == MapperPlugin.NOOP_FIELD_PREDICATE) {
+            return mappings;
+        }
+        ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(mappings.size());
+        for (ObjectObjectCursor cursor : mappings) {
+            builder.put(cursor.key, filterFields(cursor.value, fieldPredicate));
+        }
+        return builder.build(); // No types specified means return them all
+    }
+
+    @SuppressWarnings("unchecked")
+    private static MappingMetaData filterFields(MappingMetaData mappingMetaData,
+                                                Predicate fieldPredicate) throws IOException {
+        if (fieldPredicate == MapperPlugin.NOOP_FIELD_PREDICATE) {
+            return mappingMetaData;
+        }
+        Map sourceAsMap = mappingMetaData.getSourceAsMap();
+        Map properties = (Map)sourceAsMap.get("properties");
+        if (properties == null || properties.isEmpty()) {
+            return mappingMetaData;
+        }
+
+        filterFields("", properties, fieldPredicate);
+
+        return new MappingMetaData(mappingMetaData.type(), sourceAsMap);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static boolean filterFields(String currentPath, Map fields, Predicate fieldPredicate) {
+        Iterator> entryIterator = fields.entrySet().iterator();
+        while (entryIterator.hasNext()) {
+            Map.Entry entry = entryIterator.next();
+            String newPath = mergePaths(currentPath, entry.getKey());
+            Object value = entry.getValue();
+            boolean mayRemove = true;
+            boolean isMultiField = false;
+            if (value instanceof Map) {
+                Map map = (Map) value;
+                Map properties = (Map)map.get("properties");
+                if (properties != null) {
+                    mayRemove = filterFields(newPath, properties, fieldPredicate);
+                } else {
+                    Map subFields = (Map)map.get("fields");
+                    if (subFields != null) {
+                        isMultiField = true;
+                        if (mayRemove = filterFields(newPath, subFields, fieldPredicate)) {
+                            map.remove("fields");
+                        }
+                    }
+                }
+            } else {
+                throw new IllegalStateException("cannot filter mappings, found unknown element of type [" + value.getClass() + "]");
+            }
+
+            //only remove a field if it has no sub-fields left and it has to be excluded
+            if (fieldPredicate.test(newPath) == false) {
+                if (mayRemove) {
+                    entryIterator.remove();
+                } else if (isMultiField) {
+                    //multi fields that should be excluded but hold subfields that don't have to be excluded are converted to objects
+                    Map map = (Map) value;
+                    Map subFields = (Map)map.get("fields");
+                    assert subFields.size() > 0;
+                    map.put("properties", subFields);
+                    map.remove("fields");
+                    map.remove("type");
+                }
+            }
+        }
+        //return true if the ancestor may be removed, as it has no sub-fields left
+        return fields.size() == 0;
+    }
+
+    private static String mergePaths(String path, String field) {
+        if (path.length() == 0) {
+            return field;
+        }
+        return path + "." + field;
+    }
+
     /**
      * Returns all the concrete indices.
      */
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index 95c08e88898..6427368c4b9 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -74,7 +74,6 @@ public final class ThreadContext implements Closeable, Writeable {
     private static final ThreadContextStruct DEFAULT_CONTEXT = new ThreadContextStruct();
     private final Map defaultHeader;
     private final ContextThreadLocal threadLocal;
-    private boolean isSystemContext;
 
     /**
      * Creates a new ThreadContext instance
@@ -121,7 +120,6 @@ public final class ThreadContext implements Closeable, Writeable {
         return () -> threadLocal.set(context);
     }
 
-
     /**
      * Just like {@link #stashContext()} but no default context is set.
      * @param preserveResponseHeaders if set to true the response headers of the restore thread will be preserved.
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index b21f47d8feb..e34a762f527 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -105,6 +105,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         Setting.boolSetting("index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT,
                 Property.Dynamic, Property.IndexScope, Property.Deprecated);
 
+    //TODO this needs to be cleaned up: _timestamp and _ttl are not supported anymore, _field_names, _seq_no, _version and _source are
+    //also missing, not sure if on purpose. See IndicesModule#getMetadataMappers
     private static ObjectHashSet META_FIELDS = ObjectHashSet.from(
             "_uid", "_id", "_type", "_parent", "_routing", "_index",
             "_size", "_timestamp", "_ttl"
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
index e446ec7e6d3..f1e52409943 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
 import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
 import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
 import org.elasticsearch.action.resync.TransportResyncReplicationAction;
-import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
 import org.elasticsearch.common.geo.ShapesAvailability;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
@@ -33,12 +32,12 @@ import org.elasticsearch.index.mapper.BooleanFieldMapper;
 import org.elasticsearch.index.mapper.CompletionFieldMapper;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
+import org.elasticsearch.index.mapper.GeoPointFieldMapper;
 import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
 import org.elasticsearch.index.mapper.IpFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
-import org.elasticsearch.index.mapper.GeoPointFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -52,6 +51,7 @@ import org.elasticsearch.index.mapper.TypeFieldMapper;
 import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction;
+import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.flush.SyncedFlushService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
@@ -64,6 +64,9 @@ import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Predicate;
 
 /**
  * Configures classes and services that are shared by indices on each node.
@@ -73,7 +76,8 @@ public class IndicesModule extends AbstractModule {
     private final MapperRegistry mapperRegistry;
 
     public IndicesModule(List mapperPlugins) {
-        this.mapperRegistry = new MapperRegistry(getMappers(mapperPlugins), getMetadataMappers(mapperPlugins));
+        this.mapperRegistry = new MapperRegistry(getMappers(mapperPlugins), getMetadataMappers(mapperPlugins),
+                getFieldFilter(mapperPlugins));
         registerBuiltinWritables();
     }
 
@@ -118,23 +122,42 @@ public class IndicesModule extends AbstractModule {
         return Collections.unmodifiableMap(mappers);
     }
 
-    private Map getMetadataMappers(List mapperPlugins) {
+    private static final Map builtInMetadataMappers = initBuiltInMetadataMappers();
+
+    private static Map initBuiltInMetadataMappers() {
+        Map builtInMetadataMappers;
         // Use a LinkedHashMap for metadataMappers because iteration order matters
+        builtInMetadataMappers = new LinkedHashMap<>();
+        // UID first so it will be the first stored field to load (so will benefit from "fields: []" early termination
+        builtInMetadataMappers.put(UidFieldMapper.NAME, new UidFieldMapper.TypeParser());
+        builtInMetadataMappers.put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser());
+        builtInMetadataMappers.put(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser());
+        builtInMetadataMappers.put(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser());
+        builtInMetadataMappers.put(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser());
+        builtInMetadataMappers.put(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser());
+        builtInMetadataMappers.put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser());
+        builtInMetadataMappers.put(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser());
+        builtInMetadataMappers.put(SeqNoFieldMapper.NAME, new SeqNoFieldMapper.TypeParser());
+        //_field_names must be added last so that it has a chance to see all the other mappers
+        builtInMetadataMappers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser());
+        return Collections.unmodifiableMap(builtInMetadataMappers);
+    }
+
+    private static Map getMetadataMappers(List mapperPlugins) {
         Map metadataMappers = new LinkedHashMap<>();
 
-        // builtin metadata mappers
-        // UID first so it will be the first stored field to load (so will benefit from "fields: []" early termination
-
-        metadataMappers.put(UidFieldMapper.NAME, new UidFieldMapper.TypeParser());
-        metadataMappers.put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser());
-        metadataMappers.put(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser());
-        metadataMappers.put(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser());
-        metadataMappers.put(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser());
-        metadataMappers.put(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser());
-        metadataMappers.put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser());
-        metadataMappers.put(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser());
-        metadataMappers.put(SeqNoFieldMapper.NAME, new SeqNoFieldMapper.TypeParser());
-        // _field_names is not registered here, see below
+        int i = 0;
+        Map.Entry fieldNamesEntry = null;
+        for (Map.Entry entry : builtInMetadataMappers.entrySet()) {
+            if (i < builtInMetadataMappers.size() - 1) {
+                metadataMappers.put(entry.getKey(), entry.getValue());
+            } else {
+                assert entry.getKey().equals(FieldNamesFieldMapper.NAME) : "_field_names must be the last registered mapper, order counts";
+                fieldNamesEntry = entry;
+            }
+            i++;
+        }
+        assert fieldNamesEntry != null;
 
         for (MapperPlugin mapperPlugin : mapperPlugins) {
             for (Map.Entry entry : mapperPlugin.getMetadataMappers().entrySet()) {
@@ -147,11 +170,49 @@ public class IndicesModule extends AbstractModule {
             }
         }
 
-        // we register _field_names here so that it has a chance to see all other mappers, including from plugins
-        metadataMappers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser());
+        // we register _field_names here so that it has a chance to see all the other mappers, including from plugins
+        metadataMappers.put(fieldNamesEntry.getKey(), fieldNamesEntry.getValue());
         return Collections.unmodifiableMap(metadataMappers);
     }
 
+    /**
+     * Returns a set containing all of the builtin metadata fields
+     */
+    public static Set getBuiltInMetaDataFields() {
+        return builtInMetadataMappers.keySet();
+    }
+
+    private static Function> getFieldFilter(List mapperPlugins) {
+        Function> fieldFilter = MapperPlugin.NOOP_FIELD_FILTER;
+        for (MapperPlugin mapperPlugin : mapperPlugins) {
+            fieldFilter = and(fieldFilter, mapperPlugin.getFieldFilter());
+        }
+        return fieldFilter;
+    }
+
+    private static Function> and(Function> first,
+                                                           Function> second) {
+        //the purpose of this method is to not chain no-op field predicates, so that we can easily find out when no plugins plug in
+        //a field filter, hence skip the mappings filtering part as a whole, as it requires parsing mappings into a map.
+        if (first == MapperPlugin.NOOP_FIELD_FILTER) {
+            return second;
+        }
+        if (second == MapperPlugin.NOOP_FIELD_FILTER) {
+            return first;
+        }
+        return index -> {
+            Predicate firstPredicate = first.apply(index);
+            Predicate secondPredicate = second.apply(index);
+            if (firstPredicate == MapperPlugin.NOOP_FIELD_PREDICATE) {
+                return secondPredicate;
+            }
+            if (secondPredicate == MapperPlugin.NOOP_FIELD_PREDICATE) {
+                return firstPredicate;
+            }
+            return firstPredicate.and(secondPredicate);
+        };
+    }
+
     @Override
     protected void configure() {
         bind(IndicesStore.class).asEagerSingleton();
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index e2c66260a39..e6f3007a799 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -127,7 +127,9 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.function.LongSupplier;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
@@ -1262,4 +1264,22 @@ public class IndicesService extends AbstractLifecycleComponent
             }
         }
     }
+
+    /**
+     * Returns a function which given an index name, returns a predicate which fields must match in order to be returned by get mappings,
+     * get index, get field mappings and field capabilities API. Useful to filter the fields that such API return.
+     * The predicate receives the the field name as input argument. In case multiple plugins register a field filter through
+     * {@link org.elasticsearch.plugins.MapperPlugin#getFieldFilter()}, only fields that match all the registered filters will be
+     * returned by get mappings, get index, get field mappings and field capabilities API.
+     */
+    public Function> getFieldFilter() {
+        return mapperRegistry.getFieldFilter();
+    }
+
+    /**
+     * Returns true if the provided field is a registered metadata field (including ones registered via plugins), false otherwise.
+     */
+    public boolean isMetaDataField(String field) {
+        return mapperRegistry.isMetaDataField(field);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java b/core/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java
index bcc4c09d3dd..41d563c2037 100644
--- a/core/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java
+++ b/core/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java
@@ -21,10 +21,13 @@ package org.elasticsearch.indices.mapper;
 
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.plugins.MapperPlugin;
 
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.function.Function;
+import java.util.function.Predicate;
 
 /**
  * A registry for all field mappers.
@@ -33,11 +36,14 @@ public final class MapperRegistry {
 
     private final Map mapperParsers;
     private final Map metadataMapperParsers;
+    private final Function> fieldFilter;
+
 
     public MapperRegistry(Map mapperParsers,
-            Map metadataMapperParsers) {
+            Map metadataMapperParsers, Function> fieldFilter) {
         this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers));
         this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers));
+        this.fieldFilter = fieldFilter;
     }
 
     /**
@@ -55,4 +61,22 @@ public final class MapperRegistry {
     public Map getMetadataMapperParsers() {
         return metadataMapperParsers;
     }
+
+    /**
+     * Returns true if the provide field is a registered metadata field, false otherwise
+     */
+    public boolean isMetaDataField(String field) {
+        return getMetadataMapperParsers().containsKey(field);
+    }
+
+    /**
+     * Returns a function that given an index name, returns a predicate that fields must match in order to be returned by get mappings,
+     * get index, get field mappings and field capabilities API. Useful to filter the fields that such API return.
+     * The predicate receives the field name as input arguments. In case multiple plugins register a field filter through
+     * {@link MapperPlugin#getFieldFilter()}, only fields that match all the registered filters will be returned by get mappings,
+     * get index, get field mappings and field capabilities API.
+     */
+    public Function> getFieldFilter() {
+        return fieldFilter;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/MapperPlugin.java b/core/src/main/java/org/elasticsearch/plugins/MapperPlugin.java
index 5dfcdc6bda4..5edf994b32e 100644
--- a/core/src/main/java/org/elasticsearch/plugins/MapperPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/MapperPlugin.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.plugins;
 
-import java.util.Collections;
-import java.util.Map;
-
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 
+import java.util.Collections;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
 /**
  * An extension point for {@link Plugin} implementations to add custom mappers
  */
@@ -32,7 +34,7 @@ public interface MapperPlugin {
 
     /**
      * Returns additional mapper implementations added by this plugin.
-     *
+     * 
    
      * The key of the returned {@link Map} is the unique name for the mapper which will be used
      * as the mapping {@code type}, and the value is a {@link Mapper.TypeParser} to parse the
      * mapper settings into a {@link Mapper}.
@@ -43,7 +45,7 @@ public interface MapperPlugin {
 
     /**
      * Returns additional metadata mapper implementations added by this plugin.
-     *
+     * 
    
      * The key of the returned {@link Map} is the unique name for the metadata mapper, which
      * is used in the mapping json to configure the metadata mapper, and the value is a
      * {@link MetadataFieldMapper.TypeParser} to parse the mapper settings into a
@@ -52,4 +54,25 @@ public interface MapperPlugin {
     default Map getMetadataMappers() {
         return Collections.emptyMap();
     }
+
+    /**
+     * Returns a function that given an index name returns a predicate which fields must match in order to be returned by get mappings,
+     * get index, get field mappings and field capabilities API. Useful to filter the fields that such API return. The predicate receives
+     * the field name as input argument and should return true to show the field and false to hide it.
+     */
+    default Function> getFieldFilter() {
+        return NOOP_FIELD_FILTER;
+    }
+
+    /**
+     * The default field predicate applied, which doesn't filter anything. That means that by default get mappings, get index
+     * get field mappings and field capabilities API will return every field that's present in the mappings.
+     */
+    Predicate NOOP_FIELD_PREDICATE = field -> true;
+
+    /**
+     * The default field filter applied, which doesn't filter anything. That means that by default get mappings, get index
+     * get field mappings and field capabilities API will return every field that's present in the mappings.
+     */
+    Function> NOOP_FIELD_FILTER = index -> NOOP_FIELD_PREDICATE;
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java
index 0fa6831fb06..e329e70134c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java
@@ -22,6 +22,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 
@@ -31,8 +32,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testArchiveBrokenIndexSettings() {
         MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
-            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
-            Collections.emptyList());
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.emptyList());
         IndexMetaData src = newIndexMeta("foo", Settings.EMPTY);
         IndexMetaData indexMetaData = service.archiveBrokenIndexSettings(src);
         assertSame(indexMetaData, src);
@@ -59,8 +60,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testUpgrade() {
         MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
-            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
-            Collections.emptyList());
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.emptyList());
         IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build());
         assertFalse(service.isUpgraded(src));
         src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion());
@@ -72,8 +73,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testIsUpgraded() {
         MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
-            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
-            Collections.emptyList());
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.emptyList());
         IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build());
         assertFalse(service.isUpgraded(src));
         Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion());
@@ -85,8 +86,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testFailUpgrade() {
         MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
-            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
-            Collections.emptyList());
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.emptyList());
         Version minCompat = Version.CURRENT.minimumIndexCompatibilityVersion();
         Version indexUpgraded = VersionUtils.randomVersionBetween(random(), minCompat, VersionUtils.getPreviousVersion(Version.CURRENT));
         Version indexCreated = Version.fromString((minCompat.major - 1) + "." + randomInt(5) + "." + randomInt(5));
@@ -111,14 +112,13 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testPluginUpgrade() {
         MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
-            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
-            Collections.singletonList(
-                indexMetaData -> IndexMetaData.builder(indexMetaData)
-                    .settings(
-                        Settings.builder()
-                            .put(indexMetaData.getSettings())
-                            .put("index.refresh_interval", "10s")
-                    ).build()));
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.singletonList(
+                        indexMetaData -> IndexMetaData.builder(indexMetaData).settings(
+                            Settings.builder()
+                                .put(indexMetaData.getSettings())
+                                .put("index.refresh_interval", "10s")
+                        ).build()));
         IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "200s").build());
         assertFalse(service.isUpgraded(src));
         src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion());
@@ -129,12 +129,12 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testPluginUpgradeFailure() {
         MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
-            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
-            Collections.singletonList(
-                indexMetaData -> {
-                    throw new IllegalStateException("Cannot upgrade index " + indexMetaData.getIndex().getName());
-                }
-            ));
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.singletonList(
+                    indexMetaData -> {
+                        throw new IllegalStateException("Cannot upgrade index " + indexMetaData.getIndex().getName());
+                    }
+                ));
         IndexMetaData src = newIndexMeta("foo", Settings.EMPTY);
         String message = expectThrows(IllegalStateException.class, () -> service.upgradeIndexMetaData(src,
             Version.CURRENT.minimumIndexCompatibilityVersion())).getMessage();
@@ -150,7 +150,6 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
             .put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.V_5_0_0_beta1)
             .put(indexSettings)
             .build();
-        IndexMetaData metaData = IndexMetaData.builder(name).settings(build).build();
-        return metaData;
+        return IndexMetaData.builder(name).settings(build).build();
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index dd7683c1de2..54995de4b29 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -21,17 +21,21 @@ package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -231,4 +235,370 @@ public class MetaDataTests extends ESTestCase {
         );
         assertThat(fromStreamMeta.indexGraveyard(), equalTo(fromStreamMeta.indexGraveyard()));
     }
+
+    public void testFindMappings() throws IOException {
+        MetaData metaData = MetaData.builder()
+                .put(IndexMetaData.builder("index1")
+                    .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
+                    .putMapping("doc", FIND_MAPPINGS_TEST_ITEM))
+                .put(IndexMetaData.builder("index2")
+                    .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
+                    .putMapping("doc", FIND_MAPPINGS_TEST_ITEM)).build();
+
+        {
+            ImmutableOpenMap> mappings = metaData.findMappings(Strings.EMPTY_ARRAY,
+                    Strings.EMPTY_ARRAY, MapperPlugin.NOOP_FIELD_FILTER);
+            assertEquals(0, mappings.size());
+        }
+        {
+            ImmutableOpenMap> mappings = metaData.findMappings(new String[]{"index1"},
+                     new String[]{"notfound"}, MapperPlugin.NOOP_FIELD_FILTER);
+            assertEquals(0, mappings.size());
+        }
+        {
+            ImmutableOpenMap> mappings = metaData.findMappings(new String[]{"index1"},
+                    Strings.EMPTY_ARRAY, MapperPlugin.NOOP_FIELD_FILTER);
+            assertEquals(1, mappings.size());
+            assertIndexMappingsNotFiltered(mappings, "index1");
+        }
+        {
+            ImmutableOpenMap> mappings = metaData.findMappings(
+                    new String[]{"index1", "index2"},
+                    new String[]{randomBoolean() ? "doc" : "_all"}, MapperPlugin.NOOP_FIELD_FILTER);
+            assertEquals(2, mappings.size());
+            assertIndexMappingsNotFiltered(mappings, "index1");
+            assertIndexMappingsNotFiltered(mappings, "index2");
+        }
+    }
+
+    public void testFindMappingsNoOpFilters() throws IOException {
+        MappingMetaData originalMappingMetaData = new MappingMetaData("doc",
+                XContentHelper.convertToMap(JsonXContent.jsonXContent, FIND_MAPPINGS_TEST_ITEM, true));
+
+        MetaData metaData = MetaData.builder()
+                .put(IndexMetaData.builder("index1")
+                        .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
+                        .putMapping(originalMappingMetaData)).build();
+
+        {
+            ImmutableOpenMap> mappings = metaData.findMappings(new String[]{"index1"},
+                    randomBoolean() ? Strings.EMPTY_ARRAY : new String[]{"_all"}, MapperPlugin.NOOP_FIELD_FILTER);
+            ImmutableOpenMap index1 = mappings.get("index1");
+            MappingMetaData mappingMetaData = index1.get("doc");
+            assertSame(originalMappingMetaData, mappingMetaData);
+        }
+        {
+            ImmutableOpenMap> mappings = metaData.findMappings(new String[]{"index1"},
+                    randomBoolean() ? Strings.EMPTY_ARRAY : new String[]{"_all"}, index -> field -> randomBoolean());
+            ImmutableOpenMap index1 = mappings.get("index1");
+            MappingMetaData mappingMetaData = index1.get("doc");
+            assertNotSame(originalMappingMetaData, mappingMetaData);
+        }
+        {
+            ImmutableOpenMap> mappings = metaData.findMappings(new String[]{"index1"},
+                    new String[]{"doc"}, MapperPlugin.NOOP_FIELD_FILTER);
+            ImmutableOpenMap index1 = mappings.get("index1");
+            MappingMetaData mappingMetaData = index1.get("doc");
+            assertSame(originalMappingMetaData, mappingMetaData);
+        }
+        {
+            ImmutableOpenMap> mappings = metaData.findMappings(new String[]{"index1"},
+                    new String[]{"doc"}, index -> field -> randomBoolean());
+            ImmutableOpenMap index1 = mappings.get("index1");
+            MappingMetaData mappingMetaData = index1.get("doc");
+            assertNotSame(originalMappingMetaData, mappingMetaData);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testFindMappingsWithFilters() throws IOException {
+        MetaData metaData = MetaData.builder()
+                .put(IndexMetaData.builder("index1")
+                        .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
+                .putMapping("doc", FIND_MAPPINGS_TEST_ITEM))
+                .put(IndexMetaData.builder("index2")
+                        .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
+                        .putMapping("doc", FIND_MAPPINGS_TEST_ITEM))
+                .put(IndexMetaData.builder("index3")
+                        .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
+                        .putMapping("doc", FIND_MAPPINGS_TEST_ITEM)).build();
+
+        {
+            ImmutableOpenMap> mappings = metaData.findMappings(
+                    new String[]{"index1", "index2", "index3"},
+                    new String[]{"doc"}, index -> {
+                        if (index.equals("index1")) {
+                            return field -> field.startsWith("name.") == false && field.startsWith("properties.key.") == false
+                                    && field.equals("age") == false && field.equals("address.location") == false;
+                        }
+                        if (index.equals("index2")) {
+                            return field -> false;
+                        }
+                        return MapperPlugin.NOOP_FIELD_PREDICATE;
+                    });
+
+
+
+            assertIndexMappingsNoFields(mappings, "index2");
+            assertIndexMappingsNotFiltered(mappings, "index3");
+
+            ImmutableOpenMap index1Mappings = mappings.get("index1");
+            assertNotNull(index1Mappings);
+
+            assertEquals(1, index1Mappings.size());
+            MappingMetaData docMapping = index1Mappings.get("doc");
+            assertNotNull(docMapping);
+
+            Map sourceAsMap = docMapping.getSourceAsMap();
+            assertEquals(3, sourceAsMap.size());
+            assertTrue(sourceAsMap.containsKey("_routing"));
+            assertTrue(sourceAsMap.containsKey("_source"));
+
+            Map typeProperties = (Map) sourceAsMap.get("properties");
+            assertEquals(6, typeProperties.size());
+            assertTrue(typeProperties.containsKey("birth"));
+            assertTrue(typeProperties.containsKey("ip"));
+            assertTrue(typeProperties.containsKey("suggest"));
+
+            Map name = (Map) typeProperties.get("name");
+            assertNotNull(name);
+            assertEquals(1, name.size());
+            Map nameProperties = (Map) name.get("properties");
+            assertNotNull(nameProperties);
+            assertEquals(0, nameProperties.size());
+
+            Map address = (Map) typeProperties.get("address");
+            assertNotNull(address);
+            assertEquals(2, address.size());
+            assertTrue(address.containsKey("type"));
+            Map addressProperties = (Map) address.get("properties");
+            assertNotNull(addressProperties);
+            assertEquals(2, addressProperties.size());
+            assertLeafs(addressProperties, "street", "area");
+
+            Map properties = (Map) typeProperties.get("properties");
+            assertNotNull(properties);
+            assertEquals(2, properties.size());
+            assertTrue(properties.containsKey("type"));
+            Map propertiesProperties = (Map) properties.get("properties");
+            assertNotNull(propertiesProperties);
+            assertEquals(2, propertiesProperties.size());
+            assertLeafs(propertiesProperties, "key");
+            assertMultiField(propertiesProperties, "value", "keyword");
+        }
+
+        {
+            ImmutableOpenMap> mappings = metaData.findMappings(
+                    new String[]{"index1", "index2" , "index3"},
+                    new String[]{"doc"}, index -> field -> (index.equals("index3") && field.endsWith("keyword")));
+
+            assertIndexMappingsNoFields(mappings, "index1");
+            assertIndexMappingsNoFields(mappings, "index2");
+            ImmutableOpenMap index3 = mappings.get("index3");
+            assertEquals(1, index3.size());
+            MappingMetaData mappingMetaData = index3.get("doc");
+            Map sourceAsMap = mappingMetaData.getSourceAsMap();
+            assertEquals(3, sourceAsMap.size());
+            assertTrue(sourceAsMap.containsKey("_routing"));
+            assertTrue(sourceAsMap.containsKey("_source"));
+            Map typeProperties = (Map) sourceAsMap.get("properties");
+            assertNotNull(typeProperties);
+            assertEquals(1, typeProperties.size());
+            Map properties = (Map) typeProperties.get("properties");
+            assertNotNull(properties);
+            assertEquals(2, properties.size());
+            assertTrue(properties.containsKey("type"));
+            Map propertiesProperties = (Map) properties.get("properties");
+            assertNotNull(propertiesProperties);
+            assertEquals(2, propertiesProperties.size());
+            Map key = (Map) propertiesProperties.get("key");
+            assertEquals(1, key.size());
+            Map keyProperties = (Map) key.get("properties");
+            assertEquals(1, keyProperties.size());
+            assertLeafs(keyProperties, "keyword");
+            Map value = (Map) propertiesProperties.get("value");
+            assertEquals(1, value.size());
+            Map valueProperties = (Map) value.get("properties");
+            assertEquals(1, valueProperties.size());
+            assertLeafs(valueProperties, "keyword");
+        }
+
+        {
+            ImmutableOpenMap> mappings = metaData.findMappings(
+                    new String[]{"index1", "index2" , "index3"},
+                    new String[]{"doc"}, index -> field -> (index.equals("index2")));
+
+            assertIndexMappingsNoFields(mappings, "index1");
+            assertIndexMappingsNoFields(mappings, "index3");
+            assertIndexMappingsNotFiltered(mappings, "index2");
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertIndexMappingsNoFields(ImmutableOpenMap> mappings,
+                                                    String index) {
+        ImmutableOpenMap indexMappings = mappings.get(index);
+        assertNotNull(indexMappings);
+        assertEquals(1, indexMappings.size());
+        MappingMetaData docMapping = indexMappings.get("doc");
+        assertNotNull(docMapping);
+        Map sourceAsMap = docMapping.getSourceAsMap();
+        assertEquals(3, sourceAsMap.size());
+        assertTrue(sourceAsMap.containsKey("_routing"));
+        assertTrue(sourceAsMap.containsKey("_source"));
+        Map typeProperties = (Map) sourceAsMap.get("properties");
+        assertEquals(0, typeProperties.size());
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertIndexMappingsNotFiltered(ImmutableOpenMap> mappings,
+                                                       String index) {
+        ImmutableOpenMap indexMappings = mappings.get(index);
+        assertNotNull(indexMappings);
+
+        assertEquals(1, indexMappings.size());
+        MappingMetaData docMapping = indexMappings.get("doc");
+        assertNotNull(docMapping);
+
+        Map sourceAsMap = docMapping.getSourceAsMap();
+        assertEquals(3, sourceAsMap.size());
+        assertTrue(sourceAsMap.containsKey("_routing"));
+        assertTrue(sourceAsMap.containsKey("_source"));
+
+        Map typeProperties = (Map) sourceAsMap.get("properties");
+        assertEquals(7, typeProperties.size());
+        assertTrue(typeProperties.containsKey("birth"));
+        assertTrue(typeProperties.containsKey("age"));
+        assertTrue(typeProperties.containsKey("ip"));
+        assertTrue(typeProperties.containsKey("suggest"));
+
+        Map name = (Map) typeProperties.get("name");
+        assertNotNull(name);
+        assertEquals(1, name.size());
+        Map nameProperties = (Map) name.get("properties");
+        assertNotNull(nameProperties);
+        assertEquals(2, nameProperties.size());
+        assertLeafs(nameProperties, "first", "last");
+
+        Map address = (Map) typeProperties.get("address");
+        assertNotNull(address);
+        assertEquals(2, address.size());
+        assertTrue(address.containsKey("type"));
+        Map addressProperties = (Map) address.get("properties");
+        assertNotNull(addressProperties);
+        assertEquals(3, addressProperties.size());
+        assertLeafs(addressProperties, "street", "location", "area");
+
+        Map properties = (Map) typeProperties.get("properties");
+        assertNotNull(properties);
+        assertEquals(2, properties.size());
+        assertTrue(properties.containsKey("type"));
+        Map propertiesProperties = (Map) properties.get("properties");
+        assertNotNull(propertiesProperties);
+        assertEquals(2, propertiesProperties.size());
+        assertMultiField(propertiesProperties, "key", "keyword");
+        assertMultiField(propertiesProperties, "value", "keyword");
+    }
+
+    @SuppressWarnings("unchecked")
+    public static void assertLeafs(Map properties, String... fields) {
+        for (String field : fields) {
+            assertTrue(properties.containsKey(field));
+            @SuppressWarnings("unchecked")
+            Map fieldProp = (Map)properties.get(field);
+            assertNotNull(fieldProp);
+            assertFalse(fieldProp.containsKey("properties"));
+            assertFalse(fieldProp.containsKey("fields"));
+        }
+    }
+
+    public static void assertMultiField(Map properties, String field, String... subFields) {
+        assertTrue(properties.containsKey(field));
+        @SuppressWarnings("unchecked")
+        Map fieldProp = (Map)properties.get(field);
+        assertNotNull(fieldProp);
+        assertTrue(fieldProp.containsKey("fields"));
+        @SuppressWarnings("unchecked")
+        Map subFieldsDef = (Map) fieldProp.get("fields");
+        assertLeafs(subFieldsDef, subFields);
+    }
+
+    private static final String FIND_MAPPINGS_TEST_ITEM = "{\n" +
+            "  \"doc\": {\n" +
+            "      \"_routing\": {\n" +
+            "        \"required\":true\n" +
+            "      }," +
+            "      \"_source\": {\n" +
+            "        \"enabled\":false\n" +
+            "      }," +
+            "      \"properties\": {\n" +
+            "        \"name\": {\n" +
+            "          \"properties\": {\n" +
+            "            \"first\": {\n" +
+            "              \"type\": \"keyword\"\n" +
+            "            },\n" +
+            "            \"last\": {\n" +
+            "              \"type\": \"keyword\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"birth\": {\n" +
+            "          \"type\": \"date\"\n" +
+            "        },\n" +
+            "        \"age\": {\n" +
+            "          \"type\": \"integer\"\n" +
+            "        },\n" +
+            "        \"ip\": {\n" +
+            "          \"type\": \"ip\"\n" +
+            "        },\n" +
+            "        \"suggest\" : {\n" +
+            "          \"type\": \"completion\"\n" +
+            "        },\n" +
+            "        \"address\": {\n" +
+            "          \"type\": \"object\",\n" +
+            "          \"properties\": {\n" +
+            "            \"street\": {\n" +
+            "              \"type\": \"keyword\"\n" +
+            "            },\n" +
+            "            \"location\": {\n" +
+            "              \"type\": \"geo_point\"\n" +
+            "            },\n" +
+            "            \"area\": {\n" +
+            "              \"type\": \"geo_shape\",  \n" +
+            "              \"tree\": \"quadtree\",\n" +
+            "              \"precision\": \"1m\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"properties\": {\n" +
+            "          \"type\": \"nested\",\n" +
+            "          \"properties\": {\n" +
+            "            \"key\" : {\n" +
+            "              \"type\": \"text\",\n" +
+            "              \"fields\": {\n" +
+            "                \"keyword\" : {\n" +
+            "                  \"type\" : \"keyword\"\n" +
+            "                }\n" +
+            "              }\n" +
+            "            },\n" +
+            "            \"value\" : {\n" +
+            "              \"type\": \"text\",\n" +
+            "              \"fields\": {\n" +
+            "                \"keyword\" : {\n" +
+            "                  \"type\" : \"keyword\"\n" +
+            "                }\n" +
+            "              }\n" +
+            "            }\n" +
+            "          }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }\n" +
+            "  }\n" +
+            "}";
 }
diff --git a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
index 50be2516f4e..f82f2c39f44 100644
--- a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
+++ b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
@@ -39,6 +39,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
@@ -92,7 +93,7 @@ public class CodecTests extends ESTestCase {
         IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
         SimilarityService similarityService = new SimilarityService(settings, null, Collections.emptyMap());
         IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers;
-        MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap());
+        MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER);
         MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry,
                 () -> null);
         return new CodecService(service, ESLoggerFactory.getLogger("test"));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
index c878a4767b9..72d6e8c4c2c 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -60,7 +61,8 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase {
         IndexService indexService = createIndex("test", settings);
         MapperRegistry mapperRegistry = new MapperRegistry(
                 Collections.singletonMap(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")),
-                Collections.singletonMap(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser()));
+                Collections.singletonMap(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser()),
+                MapperPlugin.NOOP_FIELD_FILTER);
 
         Supplier queryShardContext = () -> {
             return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null);
@@ -111,7 +113,7 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase {
         mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo"));
         mapperParsers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser());
         mapperParsers.put(KeywordFieldMapper.CONTENT_TYPE, new KeywordFieldMapper.TypeParser());
-        MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap());
+        MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER);
 
         Supplier queryShardContext = () -> {
             return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null);
@@ -177,7 +179,7 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase {
         mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo"));
         mapperParsers.put(ExternalMapperPlugin.EXTERNAL_BIS, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "bar"));
         mapperParsers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser());
-        MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap());
+        MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER);
 
         Supplier queryShardContext = () -> {
             return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
new file mode 100644
index 00000000000..86587be951f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.junit.Before;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.cluster.metadata.MetaDataTests.assertLeafs;
+import static org.elasticsearch.cluster.metadata.MetaDataTests.assertMultiField;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase {
+
+    @Override
+    protected Collection> getPlugins() {
+        return Collections.singleton(FieldFilterPlugin.class);
+    }
+
+    @Before
+    public void putMappings() {
+        assertAcked(client().admin().indices().prepareCreate("index1"));
+        assertAcked(client().admin().indices().prepareCreate("filtered"));
+        assertAcked(client().admin().indices().preparePutMapping("index1", "filtered")
+                .setType("doc").setSource(TEST_ITEM, XContentType.JSON));
+    }
+
+    public void testGetMappings() {
+        GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings().get();
+        assertExpectedMappings(getMappingsResponse.mappings());
+    }
+
+    public void testGetIndex() {
+        GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex()
+                .setFeatures(GetIndexRequest.Feature.MAPPINGS).get();
+        assertExpectedMappings(getIndexResponse.mappings());
+    }
+
+    public void testGetFieldMappings() {
+        GetFieldMappingsResponse getFieldMappingsResponse = client().admin().indices().prepareGetFieldMappings().setFields("*").get();
+        Map>> mappings = getFieldMappingsResponse.mappings();
+        assertEquals(2, mappings.size());
+        assertFieldMappings(mappings.get("index1"), ALL_FLAT_FIELDS);
+        assertFieldMappings(mappings.get("filtered"), FILTERED_FLAT_FIELDS);
+        //double check that submitting the filtered mappings to an unfiltered index leads to the same get field mappings output
+        //as the one coming from a filtered index with same mappings
+        GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("filtered").get();
+        ImmutableOpenMap filtered = getMappingsResponse.getMappings().get("filtered");
+        assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", filtered.get("doc").getSourceAsMap()));
+        GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings("test").setFields("*").get();
+        assertEquals(1, response.mappings().size());
+        assertFieldMappings(response.mappings().get("test"), FILTERED_FLAT_FIELDS);
+    }
+
+    public void testFieldCapabilities() {
+        FieldCapabilitiesResponse index1 = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("index1")).actionGet();
+        assertFieldCaps(index1, ALL_FLAT_FIELDS);
+        FieldCapabilitiesResponse filtered = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("filtered")).actionGet();
+        assertFieldCaps(filtered, FILTERED_FLAT_FIELDS);
+        //double check that submitting the filtered mappings to an unfiltered index leads to the same field_caps output
+        //as the one coming from a filtered index with same mappings
+        GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("filtered").get();
+        ImmutableOpenMap filteredMapping = getMappingsResponse.getMappings().get("filtered");
+        assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", filteredMapping.get("doc").getSourceAsMap()));
+        FieldCapabilitiesResponse test = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("test")).actionGet();
+        assertFieldCaps(test, FILTERED_FLAT_FIELDS);
+    }
+
+    private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesResponse, String[] expectedFields) {
+        Map> responseMap = fieldCapabilitiesResponse.get();
+        Set builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
+        for (String field : builtInMetaDataFields) {
+            Map remove = responseMap.remove(field);
+            assertNotNull(" expected field [" + field + "] not found", remove);
+        }
+        for (String field : expectedFields) {
+            Map remove = responseMap.remove(field);
+            assertNotNull(" expected field [" + field + "] not found", remove);
+        }
+        assertEquals("Some unexpected fields were returned: " + responseMap.keySet(), 0, responseMap.size());
+    }
+
+    private static void assertFieldMappings(Map> mappings,
+                                            String[] expectedFields) {
+        assertEquals(1, mappings.size());
+        Map fields = new HashMap<>(mappings.get("doc"));
+        Set builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
+        for (String field : builtInMetaDataFields) {
+            GetFieldMappingsResponse.FieldMappingMetaData fieldMappingMetaData = fields.remove(field);
+            assertNotNull(" expected field [" + field + "] not found", fieldMappingMetaData);
+        }
+        for (String field : expectedFields) {
+            GetFieldMappingsResponse.FieldMappingMetaData fieldMappingMetaData = fields.remove(field);
+            assertNotNull("expected field [" + field + "] not found", fieldMappingMetaData);
+        }
+        assertEquals("Some unexpected fields were returned: " + fields.keySet(), 0, fields.size());
+    }
+
+    private void assertExpectedMappings(ImmutableOpenMap> mappings) {
+        assertEquals(2, mappings.size());
+        assertNotFiltered(mappings.get("index1"));
+        ImmutableOpenMap filtered = mappings.get("filtered");
+        assertFiltered(filtered);
+        assertMappingsAreValid(filtered.get("doc").getSourceAsMap());
+    }
+
+    private void assertMappingsAreValid(Map sourceAsMap) {
+        //check that the returned filtered mappings are still valid mappings by submitting them and retrieving them back
+        assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", sourceAsMap));
+        GetMappingsResponse testMappingsResponse = client().admin().indices().prepareGetMappings("test").get();
+        assertEquals(1, testMappingsResponse.getMappings().size());
+        //the mappings are returned unfiltered for this index, yet they are the same as the previous ones that were returned filtered
+        assertFiltered(testMappingsResponse.getMappings().get("test"));
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertFiltered(ImmutableOpenMap mappings) {
+        assertEquals(1, mappings.size());
+        MappingMetaData mappingMetaData = mappings.get("doc");
+        assertNotNull(mappingMetaData);
+        Map sourceAsMap = mappingMetaData.getSourceAsMap();
+        assertEquals(4, sourceAsMap.size());
+        assertTrue(sourceAsMap.containsKey("_meta"));
+        assertTrue(sourceAsMap.containsKey("_routing"));
+        assertTrue(sourceAsMap.containsKey("_source"));
+        Map typeProperties = (Map)sourceAsMap.get("properties");
+        assertEquals(4, typeProperties.size());
+
+        Map name = (Map)typeProperties.get("name");
+        assertEquals(1, name.size());
+        Map nameProperties = (Map)name.get("properties");
+        assertEquals(1, nameProperties.size());
+        assertLeafs(nameProperties, "last_visible");
+
+        assertLeafs(typeProperties, "age_visible");
+
+        Map address = (Map) typeProperties.get("address");
+        assertNotNull(address);
+        assertEquals(1, address.size());
+        Map addressProperties = (Map) address.get("properties");
+        assertNotNull(addressProperties);
+        assertEquals(1, addressProperties.size());
+        assertLeafs(addressProperties, "area_visible");
+
+        Map properties = (Map) typeProperties.get("properties");
+        assertNotNull(properties);
+        assertEquals(2, properties.size());
+        assertEquals("nested", properties.get("type"));
+        Map propertiesProperties = (Map) properties.get("properties");
+        assertNotNull(propertiesProperties);
+        assertEquals(2, propertiesProperties.size());
+        assertLeafs(propertiesProperties, "key_visible");
+
+        Map value = (Map) propertiesProperties.get("value");
+        assertNotNull(value);
+        assertEquals(1, value.size());
+        Map valueProperties = (Map) value.get("properties");
+        assertNotNull(valueProperties);
+        assertEquals(1, valueProperties.size());
+        assertLeafs(valueProperties, "keyword_visible");
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertNotFiltered(ImmutableOpenMap mappings) {
+        assertEquals(1, mappings.size());
+        MappingMetaData mappingMetaData = mappings.get("doc");
+        assertNotNull(mappingMetaData);
+        Map sourceAsMap = mappingMetaData.getSourceAsMap();
+        assertEquals(4, sourceAsMap.size());
+        assertTrue(sourceAsMap.containsKey("_meta"));
+        assertTrue(sourceAsMap.containsKey("_routing"));
+        assertTrue(sourceAsMap.containsKey("_source"));
+        Map typeProperties = (Map)sourceAsMap.get("properties");
+        assertEquals(5, typeProperties.size());
+
+        Map name = (Map)typeProperties.get("name");
+        assertEquals(1, name.size());
+        Map nameProperties = (Map)name.get("properties");
+        assertEquals(2, nameProperties.size());
+        assertLeafs(nameProperties, "first", "last_visible");
+
+        assertLeafs(typeProperties, "birth", "age_visible");
+
+        Map address = (Map) typeProperties.get("address");
+        assertNotNull(address);
+        assertEquals(1, address.size());
+        Map addressProperties = (Map) address.get("properties");
+        assertNotNull(addressProperties);
+        assertEquals(3, addressProperties.size());
+        assertLeafs(addressProperties, "street", "location", "area_visible");
+
+        Map properties = (Map) typeProperties.get("properties");
+        assertNotNull(properties);
+        assertEquals(2, properties.size());
+        assertTrue(properties.containsKey("type"));
+        Map propertiesProperties = (Map) properties.get("properties");
+        assertNotNull(propertiesProperties);
+        assertEquals(2, propertiesProperties.size());
+        assertMultiField(propertiesProperties, "key_visible", "keyword");
+        assertMultiField(propertiesProperties, "value", "keyword_visible");
+    }
+
+    public static class FieldFilterPlugin extends Plugin implements MapperPlugin {
+
+        @Override
+        public Function> getFieldFilter() {
+            return index -> index.equals("filtered") ? field ->  field.endsWith("visible") : MapperPlugin.NOOP_FIELD_PREDICATE;
+        }
+    }
+
+    private static final String[] ALL_FLAT_FIELDS = new String[]{
+        "name.first", "name.last_visible", "birth", "age_visible", "address.street", "address.location", "address.area_visible",
+        "properties.key_visible", "properties.key_visible.keyword", "properties.value", "properties.value.keyword_visible"
+    };
+
+    private static final String[] FILTERED_FLAT_FIELDS = new String[]{
+            "name.last_visible", "age_visible", "address.area_visible", "properties.key_visible", "properties.value.keyword_visible"
+    };
+
+    private static final String TEST_ITEM = "{\n" +
+            "  \"doc\": {\n" +
+            "      \"_meta\": {\n" +
+            "        \"version\":0.19\n" +
+            "      }," +
+            "      \"_routing\": {\n" +
+            "        \"required\":true\n" +
+            "      }," +
+            "      \"_source\": {\n" +
+            "        \"enabled\":false\n" +
+            "      }," +
+            "      \"properties\": {\n" +
+            "        \"name\": {\n" +
+            "          \"properties\": {\n" +
+            "            \"first\": {\n" +
+            "              \"type\": \"keyword\"\n" +
+            "            },\n" +
+            "            \"last_visible\": {\n" +
+            "              \"type\": \"keyword\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"birth\": {\n" +
+            "          \"type\": \"date\"\n" +
+            "        },\n" +
+            "        \"age_visible\": {\n" +
+            "          \"type\": \"integer\"\n" +
+            "        },\n" +
+            "        \"address\": {\n" +
+            "          \"type\": \"object\",\n" +
+            "          \"properties\": {\n" +
+            "            \"street\": {\n" +
+            "              \"type\": \"keyword\"\n" +
+            "            },\n" +
+            "            \"location\": {\n" +
+            "              \"type\": \"geo_point\"\n" +
+            "            },\n" +
+            "            \"area_visible\": {\n" +
+            "              \"type\": \"geo_shape\",  \n" +
+            "              \"tree\": \"quadtree\",\n" +
+            "              \"precision\": \"1m\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        },\n" +
+            "        \"properties\": {\n" +
+            "          \"type\": \"nested\",\n" +
+            "          \"properties\": {\n" +
+            "            \"key_visible\" : {\n" +
+            "              \"type\": \"text\",\n" +
+            "              \"fields\": {\n" +
+            "                \"keyword\" : {\n" +
+            "                  \"type\" : \"keyword\"\n" +
+            "                }\n" +
+            "              }\n" +
+            "            },\n" +
+            "            \"value\" : {\n" +
+            "              \"type\": \"text\",\n" +
+            "              \"fields\": {\n" +
+            "                \"keyword_visible\" : {\n" +
+            "                  \"type\" : \"keyword\"\n" +
+            "                }\n" +
+            "              }\n" +
+            "            }\n" +
+            "          }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }\n" +
+            "  }\n" +
+            "}";
+}
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
index 298bb57c499..6c4cda7fc52 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
@@ -19,23 +19,37 @@
 
 package org.elasticsearch.indices;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.IndexFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.mapper.RoutingFieldMapper;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
+import org.elasticsearch.index.mapper.TypeFieldMapper;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThan;
 
 public class IndicesModuleTests extends ESTestCase {
 
@@ -59,7 +73,7 @@ public class IndicesModuleTests extends ESTestCase {
         }
     }
 
-    List fakePlugins = Arrays.asList(new MapperPlugin() {
+    private final List fakePlugins = Arrays.asList(new MapperPlugin() {
         @Override
         public Map getMappers() {
             return Collections.singletonMap("fake-mapper", new FakeMapperParser());
@@ -70,17 +84,44 @@ public class IndicesModuleTests extends ESTestCase {
         }
     });
 
+    private static String[] EXPECTED_METADATA_FIELDS = new String[]{UidFieldMapper.NAME, IdFieldMapper.NAME, RoutingFieldMapper.NAME,
+            IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, VersionFieldMapper.NAME, ParentFieldMapper.NAME,
+            SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME};
+
     public void testBuiltinMappers() {
         IndicesModule module = new IndicesModule(Collections.emptyList());
         assertFalse(module.getMapperRegistry().getMapperParsers().isEmpty());
         assertFalse(module.getMapperRegistry().getMetadataMapperParsers().isEmpty());
+        Map metadataMapperParsers = module.getMapperRegistry().getMetadataMapperParsers();
+        int i = 0;
+        for (String field : metadataMapperParsers.keySet()) {
+            assertEquals(EXPECTED_METADATA_FIELDS[i++], field);
+        }
     }
 
     public void testBuiltinWithPlugins() {
+        IndicesModule noPluginsModule = new IndicesModule(Collections.emptyList());
         IndicesModule module = new IndicesModule(fakePlugins);
         MapperRegistry registry = module.getMapperRegistry();
-        assertThat(registry.getMapperParsers().size(), Matchers.greaterThan(1));
-        assertThat(registry.getMetadataMapperParsers().size(), Matchers.greaterThan(1));
+        assertThat(registry.getMapperParsers().size(), greaterThan(noPluginsModule.getMapperRegistry().getMapperParsers().size()));
+        assertThat(registry.getMetadataMapperParsers().size(),
+                greaterThan(noPluginsModule.getMapperRegistry().getMetadataMapperParsers().size()));
+        Map metadataMapperParsers = module.getMapperRegistry().getMetadataMapperParsers();
+        Iterator iterator = metadataMapperParsers.keySet().iterator();
+        assertEquals(UidFieldMapper.NAME, iterator.next());
+        String last = null;
+        while(iterator.hasNext()) {
+            last = iterator.next();
+        }
+        assertEquals(FieldNamesFieldMapper.NAME, last);
+    }
+
+    public void testGetBuiltInMetaDataFields() {
+        Set builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
+        int i = 0;
+        for (String field : builtInMetaDataFields) {
+            assertEquals(EXPECTED_METADATA_FIELDS[i++], field);
+        }
     }
 
     public void testDuplicateBuiltinMapper() {
@@ -92,7 +133,7 @@ public class IndicesModuleTests extends ESTestCase {
         });
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
             () -> new IndicesModule(plugins));
-        assertThat(e.getMessage(), Matchers.containsString("already registered"));
+        assertThat(e.getMessage(), containsString("already registered"));
     }
 
     public void testDuplicateOtherPluginMapper() {
@@ -105,7 +146,7 @@ public class IndicesModuleTests extends ESTestCase {
         List plugins = Arrays.asList(plugin, plugin);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
             () -> new IndicesModule(plugins));
-        assertThat(e.getMessage(), Matchers.containsString("already registered"));
+        assertThat(e.getMessage(), containsString("already registered"));
     }
 
     public void testDuplicateBuiltinMetadataMapper() {
@@ -117,7 +158,7 @@ public class IndicesModuleTests extends ESTestCase {
         });
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
             () -> new IndicesModule(plugins));
-        assertThat(e.getMessage(), Matchers.containsString("already registered"));
+        assertThat(e.getMessage(), containsString("already registered"));
     }
 
     public void testDuplicateOtherPluginMetadataMapper() {
@@ -130,7 +171,7 @@ public class IndicesModuleTests extends ESTestCase {
         List plugins = Arrays.asList(plugin, plugin);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
             () -> new IndicesModule(plugins));
-        assertThat(e.getMessage(), Matchers.containsString("already registered"));
+        assertThat(e.getMessage(), containsString("already registered"));
     }
 
     public void testDuplicateFieldNamesMapper() {
@@ -142,20 +183,102 @@ public class IndicesModuleTests extends ESTestCase {
         });
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
             () -> new IndicesModule(plugins));
-        assertThat(e.getMessage(), Matchers.containsString("cannot contain metadata mapper [_field_names]"));
+        assertThat(e.getMessage(), containsString("cannot contain metadata mapper [_field_names]"));
     }
 
     public void testFieldNamesIsLast() {
         IndicesModule module = new IndicesModule(Collections.emptyList());
-        List fieldNames = module.getMapperRegistry().getMetadataMapperParsers().keySet()
-            .stream().collect(Collectors.toList());
+        List fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers().keySet());
         assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1));
     }
 
     public void testFieldNamesIsLastWithPlugins() {
         IndicesModule module = new IndicesModule(fakePlugins);
-        List fieldNames = module.getMapperRegistry().getMetadataMapperParsers().keySet()
-            .stream().collect(Collectors.toList());
+        List fieldNames = new ArrayList<>(module.getMapperRegistry().getMetadataMapperParsers().keySet());
         assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1));
     }
+
+    public void testGetFieldFilter() {
+        List mapperPlugins = Arrays.asList(
+            new MapperPlugin() {
+                @Override
+                public Function> getFieldFilter() {
+                    return MapperPlugin.NOOP_FIELD_FILTER;
+                }
+            },
+            new MapperPlugin() {
+                @Override
+                public Function> getFieldFilter() {
+                    return index -> index.equals("hidden_index") ? field -> false : MapperPlugin.NOOP_FIELD_PREDICATE;
+                }
+            },
+            new MapperPlugin() {
+                @Override
+                public Function> getFieldFilter() {
+                    return index -> field -> field.equals("hidden_field") == false;
+                }
+            },
+            new MapperPlugin() {
+                @Override
+                public Function> getFieldFilter() {
+                    return index -> index.equals("filtered") ? field ->  field.equals("visible") : MapperPlugin.NOOP_FIELD_PREDICATE;
+                }
+            });
+
+        IndicesModule indicesModule = new IndicesModule(mapperPlugins);
+        MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
+        Function> fieldFilter = mapperRegistry.getFieldFilter();
+        assertNotSame(MapperPlugin.NOOP_FIELD_FILTER, fieldFilter);
+
+        assertFalse(fieldFilter.apply("hidden_index").test(randomAlphaOfLengthBetween(3, 5)));
+        assertTrue(fieldFilter.apply(randomAlphaOfLengthBetween(3, 5)).test(randomAlphaOfLengthBetween(3, 5)));
+
+        assertFalse(fieldFilter.apply(randomAlphaOfLengthBetween(3, 5)).test("hidden_field"));
+        assertFalse(fieldFilter.apply("filtered").test(randomAlphaOfLengthBetween(3, 5)));
+        assertFalse(fieldFilter.apply("filtered").test("hidden_field"));
+        assertTrue(fieldFilter.apply("filtered").test("visible"));
+        assertFalse(fieldFilter.apply("hidden_index").test("visible"));
+        assertTrue(fieldFilter.apply(randomAlphaOfLengthBetween(3, 5)).test("visible"));
+        assertFalse(fieldFilter.apply("hidden_index").test("hidden_field"));
+    }
+
+    public void testDefaultFieldFilterIsNoOp() {
+        int numPlugins = randomIntBetween(0, 10);
+        List mapperPlugins = new ArrayList<>(numPlugins);
+        for (int i = 0; i < numPlugins; i++) {
+            mapperPlugins.add(new MapperPlugin() {});
+        }
+        IndicesModule indicesModule = new IndicesModule(mapperPlugins);
+        Function> fieldFilter = indicesModule.getMapperRegistry().getFieldFilter();
+        assertSame(MapperPlugin.NOOP_FIELD_FILTER, fieldFilter);
+    }
+
+    public void testNoOpFieldPredicate() {
+        List mapperPlugins = Arrays.asList(
+                new MapperPlugin() {
+                    @Override
+                    public Function> getFieldFilter() {
+                        return MapperPlugin.NOOP_FIELD_FILTER;
+                    }
+                },
+                new MapperPlugin() {
+                    @Override
+                    public Function> getFieldFilter() {
+                        return index -> index.equals("hidden_index") ? field -> false : MapperPlugin.NOOP_FIELD_PREDICATE;
+                    }
+                },
+                new MapperPlugin() {
+                    @Override
+                    public Function> getFieldFilter() {
+                        return index -> index.equals("filtered") ? field ->  field.equals("visible") : MapperPlugin.NOOP_FIELD_PREDICATE;
+                    }
+                });
+
+        IndicesModule indicesModule = new IndicesModule(mapperPlugins);
+        MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
+        Function> fieldFilter = mapperRegistry.getFieldFilter();
+        assertSame(MapperPlugin.NOOP_FIELD_PREDICATE, fieldFilter.apply(randomAlphaOfLengthBetween(3, 7)));
+        assertNotSame(MapperPlugin.NOOP_FIELD_PREDICATE, fieldFilter.apply("hidden_index"));
+        assertNotSame(MapperPlugin.NOOP_FIELD_PREDICATE, fieldFilter.apply("filtered"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index 267253ff12a..01f1c3dab9c 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -110,7 +110,6 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
         }
     }
 
-
     @Override
     protected boolean resetNodeAfterTest() {
         return true;
@@ -431,4 +430,12 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
         assertThat("index not defined", indexStats.containsKey(index), equalTo(true));
         assertThat("unexpected shard stats", indexStats.get(index), equalTo(shardStats));
     }
+
+    public void testIsMetaDataField() {
+        IndicesService indicesService = getIndicesService();
+        assertFalse(indicesService.isMetaDataField(randomAlphaOfLengthBetween(10, 15)));
+        for (String builtIn : IndicesModule.getBuiltInMetaDataFields()) {
+            assertTrue(indicesService.isMetaDataField(builtIn));
+        }
+    }
 }
diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
index d28a47cb318..ad5f06b6ee3 100644
--- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
+++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
@@ -33,6 +33,7 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -56,7 +57,7 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase {
         indexService = createIndex("test");
         mapperRegistry = new MapperRegistry(
                 Collections.singletonMap(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()),
-                Collections.emptyMap());
+                Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER);
         Supplier queryShardContext = () -> {
             return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null);
         };
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldFilterPlugin.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldFilterPlugin.java
new file mode 100644
index 00000000000..b109520ecc2
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldFilterPlugin.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+public class MockFieldFilterPlugin extends Plugin implements MapperPlugin {
+
+    @Override
+    public Function> getFieldFilter() {
+        //this filter doesn't filter any field out, but it's used to exercise the code path executed when the filter is not no-op
+        return index -> field -> true;
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index eb9998d4208..1dbf4015546 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -121,6 +121,7 @@ import org.elasticsearch.index.MockEngineFactoryPlugin;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.engine.Segment;
 import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MockFieldFilterPlugin;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.IndexShard;
@@ -1906,6 +1907,9 @@ public abstract class ESIntegTestCase extends ESTestCase {
             if (randomBoolean()) {
                 mocks.add(AssertingTransportInterceptor.TestPlugin.class);
             }
+            if (randomBoolean()) {
+                mocks.add(MockFieldFilterPlugin.class);
+            }
         }
 
         if (addMockTransportService()) {

From 2f9a88206141ad408a7de0558df9920b1a467111 Mon Sep 17 00:00:00 2001
From: Deb Adair 
Date: Tue, 5 Dec 2017 11:46:40 -0800
Subject: [PATCH 189/297] [DOCS] Fixed typos and broken attribute.

---
 docs/reference/migration/migrate_7_0.asciidoc   | 5 ++---
 docs/reference/search/request/sort.asciidoc     | 2 +-
 docs/reference/upgrade/reindex_upgrade.asciidoc | 3 +--
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/docs/reference/migration/migrate_7_0.asciidoc b/docs/reference/migration/migrate_7_0.asciidoc
index 7d84077ab86..c418a333a8d 100644
--- a/docs/reference/migration/migrate_7_0.asciidoc
+++ b/docs/reference/migration/migrate_7_0.asciidoc
@@ -12,12 +12,11 @@ Elasticsearch 7.0 node will not start in the presence of indices created in a
 version of Elasticsearch before 6.0.
 
 [IMPORTANT]
-.Reindex indices from Elasticseach 5.x or before
+.Reindex indices from Elasticsearch 5.x or before
 =========================================
 
 Indices created in Elasticsearch 5.x or before will need to be reindexed with
-Elasticsearch 6.x in order to be readable by Elasticsearch 7.x. The easiest
-way to reindex old indices is to use the `reindex` API.
+Elasticsearch 6.x in order to be readable by Elasticsearch 7.x.
 
 =========================================
 
diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc
index 226782d9f57..3549e18dbb0 100644
--- a/docs/reference/search/request/sort.asciidoc
+++ b/docs/reference/search/request/sort.asciidoc
@@ -132,7 +132,7 @@ field support has a `nested` sort option with the following properties:
     current nested object.
 
 [WARNING]
-.Nested sort options before Elasticseach 6.1
+.Nested sort options before Elasticsearch 6.1
 ============================================
 
 The `nested_path` and `nested_filter` options have been deprecated in
diff --git a/docs/reference/upgrade/reindex_upgrade.asciidoc b/docs/reference/upgrade/reindex_upgrade.asciidoc
index dc3c72fee22..c13343b64c3 100644
--- a/docs/reference/upgrade/reindex_upgrade.asciidoc
+++ b/docs/reference/upgrade/reindex_upgrade.asciidoc
@@ -75,7 +75,6 @@ To manually reindex your old indices with the <>:
 .. Add any aliases that existed on the old index to the new index.
 
 
-// Flag this as X-Pack and conditionally include at GA.
 // Need to update the CSS to override sidebar titles.
 [role="xpack"]
 .Migration assistance and upgrade tools
@@ -84,7 +83,7 @@ To manually reindex your old indices with the <>:
 reindexing and upgrading to 6.x. These tools are free with the X-Pack trial
 and Basic licenses and you can use them to upgrade whether or not X-Pack is a
 regular part of your Elastic Stack. For more information, see
-{stack-guide}/upgrading-elastic-stack.html.
+{stack-ref}/upgrading-elastic-stack.html.
 *******************************************
 
 [[reindex-upgrade-remote]]

From ed2caf2badb70353ba6265981d8c908697507456 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen 
Date: Tue, 5 Dec 2017 15:38:33 -0500
Subject: [PATCH 190/297] Prevent constructing index template without patterns
 (#27662)

Today, we prevent the system from storing a broken index template in the
transport layer, however we don't prevent this in XContent. A broken
index template can break the whole cluster state.

This commit attempts to prevent the system from constructing an index
template without a proper index patterns.
---
 .../metadata/IndexTemplateMetaData.java       |  5 ++-
 .../metadata/IndexTemplateMetaDataTests.java  | 42 +++++++++++++++++++
 .../metadata/TemplateUpgradeServiceTests.java | 27 +++++++-----
 .../ClusterSerializationTests.java            |  5 ++-
 .../ClusterStateToStringTests.java            |  5 ++-
 .../gateway/GatewayMetaStateTests.java        | 15 ++++---
 6 files changed, 80 insertions(+), 19 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
index 66f5a49f6d6..74233b5cec7 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
@@ -93,6 +93,9 @@ public class IndexTemplateMetaData extends AbstractDiffable mappings,
                                  ImmutableOpenMap aliases,
                                  ImmutableOpenMap customs) {
+        if (patterns == null || patterns.isEmpty()) {
+            throw new IllegalArgumentException("Index patterns must not be null or empty; got " + patterns);
+        }
         this.name = name;
         this.order = order;
         this.version = version;
@@ -244,7 +247,7 @@ public class IndexTemplateMetaData extends AbstractDiffable 0 ? patterns.get(0) : "");
+            out.writeString(patterns.get(0));
         }
         Settings.writeSettingsToStream(settings, out);
         out.writeVInt(mappings.size());
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
index 8f247abcf33..d5f441436e7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.cluster.metadata;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
@@ -120,4 +121,45 @@ public class IndexTemplateMetaDataTests extends ESTestCase {
         assertThat(indexTemplateMetaData, equalTo(indexTemplateMetaDataRoundTrip));
     }
 
+    public void testValidateInvalidIndexPatterns() throws Exception {
+        final IllegalArgumentException emptyPatternError = expectThrows(IllegalArgumentException.class, () -> {
+            new IndexTemplateMetaData(randomRealisticUnicodeOfLengthBetween(5, 10), randomInt(), randomInt(),
+                Collections.emptyList(), Settings.EMPTY, ImmutableOpenMap.of(), ImmutableOpenMap.of(), ImmutableOpenMap.of());
+        });
+        assertThat(emptyPatternError.getMessage(), equalTo("Index patterns must not be null or empty; got []"));
+
+        final IllegalArgumentException nullPatternError = expectThrows(IllegalArgumentException.class, () -> {
+            new IndexTemplateMetaData(randomRealisticUnicodeOfLengthBetween(5, 10), randomInt(), randomInt(),
+                null, Settings.EMPTY, ImmutableOpenMap.of(), ImmutableOpenMap.of(), ImmutableOpenMap.of());
+        });
+        assertThat(nullPatternError.getMessage(), equalTo("Index patterns must not be null or empty; got null"));
+
+        final String templateWithEmptyPattern = "{\"index_patterns\" : [],\"order\" : 1000," +
+            "\"settings\" : {\"number_of_shards\" : 10,\"number_of_replicas\" : 1}," +
+            "\"mappings\" : {\"doc\" :" +
+            "{\"properties\":{\"" +
+            randomAlphaOfLength(10) + "\":{\"type\":\"text\"},\"" +
+            randomAlphaOfLength(10) + "\":{\"type\":\"keyword\"}}" +
+            "}}}";
+        try (XContentParser parser =
+                 XContentHelper.createParser(NamedXContentRegistry.EMPTY, new BytesArray(templateWithEmptyPattern), XContentType.JSON)) {
+            final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
+            assertThat(ex.getMessage(), equalTo("Index patterns must not be null or empty; got []"));
+        }
+
+        final String templateWithoutPattern = "{\"order\" : 1000," +
+            "\"settings\" : {\"number_of_shards\" : 10,\"number_of_replicas\" : 1}," +
+            "\"mappings\" : {\"doc\" :" +
+            "{\"properties\":{\"" +
+            randomAlphaOfLength(10) + "\":{\"type\":\"text\"},\"" +
+            randomAlphaOfLength(10) + "\":{\"type\":\"keyword\"}}" +
+            "}}}";
+        try (XContentParser parser =
+                 XContentHelper.createParser(NamedXContentRegistry.EMPTY, new BytesArray(templateWithoutPattern), XContentType.JSON)) {
+            final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
+            assertThat(ex.getMessage(), equalTo("Index patterns must not be null or empty; got null"));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
index e1763fa6a5d..2e82397767f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
@@ -54,6 +54,8 @@ import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
@@ -61,9 +63,7 @@ import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doAnswer;
@@ -83,16 +83,17 @@ public class TemplateUpgradeServiceTests extends ESTestCase {
         boolean shouldChange = randomBoolean();
 
         MetaData metaData = randomMetaData(
-            IndexTemplateMetaData.builder("user_template").build(),
-            IndexTemplateMetaData.builder("removed_test_template").build(),
-            IndexTemplateMetaData.builder("changed_test_template").build()
+            IndexTemplateMetaData.builder("user_template").patterns(randomIndexPatterns()).build(),
+            IndexTemplateMetaData.builder("removed_test_template").patterns(randomIndexPatterns()).build(),
+            IndexTemplateMetaData.builder("changed_test_template").patterns(randomIndexPatterns()).build()
         );
 
         TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, null, clusterService, null,
             Arrays.asList(
                 templates -> {
                     if (shouldAdd) {
-                        assertNull(templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template").build()));
+                        assertNull(templates.put("added_test_template",
+                            IndexTemplateMetaData.builder("added_test_template").patterns(randomIndexPatterns()).build()));
                     }
                     return templates;
                 },
@@ -105,7 +106,7 @@ public class TemplateUpgradeServiceTests extends ESTestCase {
                 templates -> {
                     if (shouldChange) {
                         assertNotNull(templates.put("changed_test_template",
-                            IndexTemplateMetaData.builder("changed_test_template").order(10).build()));
+                            IndexTemplateMetaData.builder("changed_test_template").patterns(randomIndexPatterns()).order(10).build()));
                     }
                     return templates;
                 }
@@ -234,9 +235,9 @@ public class TemplateUpgradeServiceTests extends ESTestCase {
         AtomicInteger updateInvocation = new AtomicInteger();
 
         MetaData metaData = randomMetaData(
-            IndexTemplateMetaData.builder("user_template").build(),
-            IndexTemplateMetaData.builder("removed_test_template").build(),
-            IndexTemplateMetaData.builder("changed_test_template").build()
+            IndexTemplateMetaData.builder("user_template").patterns(randomIndexPatterns()).build(),
+            IndexTemplateMetaData.builder("removed_test_template").patterns(randomIndexPatterns()).build(),
+            IndexTemplateMetaData.builder("changed_test_template").patterns(randomIndexPatterns()).build()
         );
 
         ThreadPool threadPool = mock(ThreadPool.class);
@@ -390,4 +391,10 @@ public class TemplateUpgradeServiceTests extends ESTestCase {
         }
         return builder.build();
     }
+
+    List randomIndexPatterns() {
+        return IntStream.range(0, between(1, 10))
+            .mapToObj(n -> randomUnicodeOfCodepointLengthBetween(1, 100))
+            .collect(Collectors.toList());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
index a892b2a2934..d64b4a66ee7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
@@ -46,6 +46,7 @@ import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -170,8 +171,8 @@ public class ClusterSerializationTests extends ESAllocationTestCase {
     public void testObjectReuseWhenApplyingClusterStateDiff() throws Exception {
         IndexMetaData indexMetaData
             = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1).build();
-        IndexTemplateMetaData indexTemplateMetaData
-            = IndexTemplateMetaData.builder("test-template").patterns(new ArrayList<>()).build();
+        IndexTemplateMetaData indexTemplateMetaData = IndexTemplateMetaData.builder("test-template")
+            .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false))).build();
         MetaData metaData = MetaData.builder().put(indexMetaData, true).put(indexTemplateMetaData).build();
 
         RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
index 65d780f3fd9..86c6d0e02eb 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
@@ -32,6 +32,8 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 
+import java.util.Arrays;
+
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.hamcrest.Matchers.containsString;
@@ -40,7 +42,8 @@ public class ClusterStateToStringTests extends ESAllocationTestCase {
     public void testClusterStateSerialization() throws Exception {
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test_idx").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1))
-                .put(IndexTemplateMetaData.builder("test_template").build())
+                .put(IndexTemplateMetaData.builder("test_template")
+                    .patterns(Arrays.asList(generateRandomStringArray(10, 100, false,false))).build())
                 .build();
 
         RoutingTable routingTable = RoutingTable.builder()
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
index e7daa9a791d..cef3502a077 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
@@ -308,7 +308,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
             Collections.emptyList(),
             Collections.singletonList(
                 templates -> {
-                    templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template").build());
+                    templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template")
+                        .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false))).build());
                     return templates;
                 }
             ));
@@ -438,14 +439,17 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
             Collections.emptyList(),
             Arrays.asList(
                 indexTemplateMetaDatas -> {
-                    indexTemplateMetaDatas.put("template1", IndexTemplateMetaData.builder("template1").settings(
-                        Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 20).build()).build());
+                    indexTemplateMetaDatas.put("template1", IndexTemplateMetaData.builder("template1")
+                        .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false)))
+                        .settings(Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 20).build())
+                        .build());
                     return indexTemplateMetaDatas;
 
                 },
                 indexTemplateMetaDatas -> {
-                    indexTemplateMetaDatas.put("template2", IndexTemplateMetaData.builder("template2").settings(
-                        Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 10).build()).build());
+                    indexTemplateMetaDatas.put("template2", IndexTemplateMetaData.builder("template2")
+                        .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false)))
+                        .settings(Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 10).build()).build());
                     return indexTemplateMetaDatas;
 
                 }
@@ -535,6 +539,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
                 .settings(settings(Version.CURRENT)
                     .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 3))
                     .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 5)))
+                .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false)))
                 .build();
             builder.put(templateMetaData);
         }

From 8139e3a1c7aa70e0f0c8363dcc22bbdbd3ef93c4 Mon Sep 17 00:00:00 2001
From: Ryan Ernst 
Date: Tue, 5 Dec 2017 14:30:36 -0800
Subject: [PATCH 191/297] Add validation of keystore setting names (#27626)

This commit restricts settings added to the keystore to have a lowercase
ascii name. The java Keystore javadocs state that case sensitivity of
key alias names are implementation dependent. This ensures regardless of
case sensitivity in a jvm implementation, the keys will be stored as we
expect.
---
 .../common/settings/KeyStoreWrapper.java      | 21 ++++++++++++++++++-
 .../common/settings/SecureSetting.java        |  1 +
 .../common/settings/KeyStoreWrapperTests.java | 10 +++++++++
 .../common/settings/SettingsTests.java        |  9 ++++++++
 4 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
index 441bb131f03..6ebc47c8252 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
@@ -49,6 +49,7 @@ import java.util.HashSet;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import org.apache.lucene.codecs.CodecUtil;
@@ -59,7 +60,6 @@ import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.SimpleFSDirectory;
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.bootstrap.BootstrapSettings;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.Randomness;
@@ -75,6 +75,11 @@ import org.elasticsearch.common.Randomness;
  */
 public class KeyStoreWrapper implements SecureSettings {
 
+    /**
+     * A regex for the valid characters that a setting name in the keystore may use.
+     */
+    private static final Pattern ALLOWED_SETTING_NAME = Pattern.compile("[a-z0-9_\\-.]+");
+
     public static final Setting SEED_SETTING = SecureSetting.secureString("keystore.seed", null);
 
     /** Characters that may be used in the bootstrap seed setting added to all keystores. */
@@ -383,6 +388,18 @@ public class KeyStoreWrapper implements SecureSettings {
         return Base64.getDecoder().wrap(bytesStream);
     }
 
+    /**
+     * Ensure the given setting name is allowed.
+     *
+     * @throws IllegalArgumentException if the setting name is not valid
+     */
+    public static void validateSettingName(String setting) {
+        if (ALLOWED_SETTING_NAME.matcher(setting).matches() == false) {
+            throw new IllegalArgumentException("Setting name [" + setting + "] does not match the allowed setting name pattern ["
+                + ALLOWED_SETTING_NAME.pattern() + "]");
+        }
+    }
+
     /**
      * Set a string setting.
      *
@@ -390,6 +407,7 @@ public class KeyStoreWrapper implements SecureSettings {
      */
     void setString(String setting, char[] value) throws GeneralSecurityException {
         assert isLoaded();
+        validateSettingName(setting);
         if (ASCII_ENCODER.canEncode(CharBuffer.wrap(value)) == false) {
             throw new IllegalArgumentException("Value must be ascii");
         }
@@ -401,6 +419,7 @@ public class KeyStoreWrapper implements SecureSettings {
     /** Set a file setting. */
     void setFile(String setting, byte[] bytes) throws GeneralSecurityException {
         assert isLoaded();
+        validateSettingName(setting);
         bytes = Base64.getEncoder().encode(bytes);
         char[] chars = new char[bytes.length];
         for (int i = 0; i < chars.length; ++i) {
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java b/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
index 4a1e598bba8..c23a0bd42e3 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
@@ -46,6 +46,7 @@ public abstract class SecureSetting extends Setting {
     private SecureSetting(String key, Property... properties) {
         super(key, (String)null, null, ArrayUtils.concat(properties, FIXED_PROPERTIES, Property.class));
         assert assertAllowedProperties(properties);
+        KeyStoreWrapper.validateSettingName(key);
     }
 
     private boolean assertAllowedProperties(Setting.Property... properties) {
diff --git a/core/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/core/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
index 11d1e1f5735..c3b34b7c3ef 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
@@ -97,4 +97,14 @@ public class KeyStoreWrapperTests extends ESTestCase {
         keystore.decrypt(new char[0]);
         assertEquals(seed.toString(), keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey()).toString());
     }
+
+    public void testIllegalSettingName() throws Exception {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> KeyStoreWrapper.validateSettingName("UpperCase"));
+        assertTrue(e.getMessage().contains("does not match the allowed setting name pattern"));
+        KeyStoreWrapper keystore = KeyStoreWrapper.create(new char[0]);
+        e = expectThrows(IllegalArgumentException.class, () -> keystore.setString("UpperCase", new char[0]));
+        assertTrue(e.getMessage().contains("does not match the allowed setting name pattern"));
+        e = expectThrows(IllegalArgumentException.class, () -> keystore.setFile("UpperCase", new byte[0]));
+        assertTrue(e.getMessage().contains("does not match the allowed setting name pattern"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
index 04cd1717e7f..039de112fac 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
@@ -486,6 +486,15 @@ public class SettingsTests extends ESTestCase {
         assertTrue(e.getMessage().contains("must be stored inside the Elasticsearch keystore"));
     }
 
+    public void testSecureSettingIllegalName() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            SecureSetting.secureString("UpperCaseSetting", null));
+        assertTrue(e.getMessage().contains("does not match the allowed setting name pattern"));
+        e = expectThrows(IllegalArgumentException.class, () ->
+            SecureSetting.secureFile("UpperCaseSetting", null));
+        assertTrue(e.getMessage().contains("does not match the allowed setting name pattern"));
+    }
+
     public void testGetAsArrayFailsOnDuplicates() {
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> Settings.builder()
             .put("foobar.0", "bar")

From e584c520174fe6a656dae070dd968630dfe396e9 Mon Sep 17 00:00:00 2001
From: Catalin Ursachi 
Date: Tue, 5 Dec 2017 22:39:01 +0000
Subject: [PATCH 192/297] Clarify IntelliJ IDEA Jar Hell fix (#27635)

---
 CONTRIBUTING.md | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a0655b4b849..6bbb655a18b 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -111,19 +111,18 @@ then `File->New Project From Existing Sources`. Point to the root of
 the source directory, select
 `Import project from external model->Gradle`, enable
 `Use auto-import`. In order to run tests directly from
-IDEA 2017.2 and above it is required to disable IDEA run launcher to avoid
-finding yourself in "jar hell", which can be achieved by adding the
+IDEA 2017.2 and above, it is required to disable the IDEA run launcher in order to avoid
+`idea_rt.jar` causing "jar hell". This can be achieved by adding the
 `-Didea.no.launcher=true` [JVM
-option](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties)
-or by adding `idea.no.launcher=true` to the
+option](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties).
+Alternatively, `idea.no.launcher=true` can be set in the
 [`idea.properties`](https://www.jetbrains.com/help/idea/file-idea-properties.html)
-file which can be accessed under Help > Edit Custom Properties within IDEA. You
-may also need to [remove `ant-javafx.jar` from your
+file which can be accessed under Help > Edit Custom Properties (this will require a
+restart of IDEA). For IDEA 2017.3 and above, in addition to the JVM option, you will need to go to
+`Run->Edit Configurations...` and change the value for the `Shorten command line` setting from
+`user-local default: none` to `classpath file`. You may also need to [remove `ant-javafx.jar` from your
 classpath](https://github.com/elastic/elasticsearch/issues/14348) if that is
-reported as a source of jar hell. Additionally, in order to run tests directly
-from IDEA 2017.3 and above, go to `Run->Edit Configurations...` and change the
-value for the `Shorten command line` setting from `user-local default: none` to
-`classpath file`.
+reported as a source of jar hell.
 
 The Elasticsearch codebase makes heavy use of Java `assert`s and the
 test runner requires that assertions be enabled within the JVM. This

From c51e48bec004c4117cb2df59ec31cb04684e2f92 Mon Sep 17 00:00:00 2001
From: Ryan Ernst 
Date: Tue, 5 Dec 2017 15:10:18 -0800
Subject: [PATCH 193/297] Correct docs for binary fields and their default for
 doc values (#27680)

closes #27240
---
 docs/reference/mapping/types/binary.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/mapping/types/binary.asciidoc b/docs/reference/mapping/types/binary.asciidoc
index 556fc55b7f0..0633c29801e 100644
--- a/docs/reference/mapping/types/binary.asciidoc
+++ b/docs/reference/mapping/types/binary.asciidoc
@@ -43,7 +43,7 @@ The following parameters are accepted by `binary` fields:
 
     Should the field be stored on disk in a column-stride fashion, so that it
     can later be used for sorting, aggregations, or scripting? Accepts `true`
-    (default) or `false`.
+    or `false` (default).
 
 <>::
 

From 234e09a1058c2ac1dd354018d758131066cec110 Mon Sep 17 00:00:00 2001
From: javanna 
Date: Wed, 6 Dec 2017 01:42:08 +0100
Subject: [PATCH 194/297] Fix UpdateMappingIntegrationIT test failures

The mappings can be submitted wrapped in a type object or not. They need to be returned in the same way as they were submitted. When applying field filters, we need to make sure that the format is preserved. MappingMetaData#getSourceAsMap removes the root level if it's the type object, which would make us overwrite the original mappings with filtered mappings but without the original root object.

Closes #27678
---
 .../cluster/metadata/MetaData.java            | 16 +++-
 .../cluster/metadata/MetaDataTests.java       | 16 +++-
 .../mapping/UpdateMappingIntegrationIT.java   | 77 +++++++++----------
 3 files changed, 61 insertions(+), 48 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index 0c543d5ffe3..0e9bcf8f11a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -48,6 +48,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.MetaDataStateFormat;
@@ -382,13 +383,19 @@ public class MetaData implements Iterable, Diffable, To
     }
 
     @SuppressWarnings("unchecked")
-    private static MappingMetaData filterFields(MappingMetaData mappingMetaData,
-                                                Predicate fieldPredicate) throws IOException {
+    private static MappingMetaData filterFields(MappingMetaData mappingMetaData, Predicate fieldPredicate) throws IOException {
         if (fieldPredicate == MapperPlugin.NOOP_FIELD_PREDICATE) {
             return mappingMetaData;
         }
-        Map sourceAsMap = mappingMetaData.getSourceAsMap();
-        Map properties = (Map)sourceAsMap.get("properties");
+        Map sourceAsMap = XContentHelper.convertToMap(mappingMetaData.source().compressedReference(), true).v2();
+        Map mapping;
+        if (sourceAsMap.size() == 1 && sourceAsMap.containsKey(mappingMetaData.type())) {
+            mapping = (Map) sourceAsMap.get(mappingMetaData.type());
+        } else {
+            mapping = sourceAsMap;
+        }
+
+        Map properties = (Map)mapping.get("properties");
         if (properties == null || properties.isEmpty()) {
             return mappingMetaData;
         }
@@ -400,6 +407,7 @@ public class MetaData implements Iterable, Diffable, To
 
     @SuppressWarnings("unchecked")
     private static boolean filterFields(String currentPath, Map fields, Predicate fieldPredicate) {
+        assert fieldPredicate != MapperPlugin.NOOP_FIELD_PREDICATE;
         Iterator> entryIterator = fields.entrySet().iterator();
         while (entryIterator.hasNext()) {
             Map.Entry entry = entryIterator.next();
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index 54995de4b29..acf6525f99d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -315,19 +315,29 @@ public class MetaDataTests extends ESTestCase {
 
     @SuppressWarnings("unchecked")
     public void testFindMappingsWithFilters() throws IOException {
+        String mapping = FIND_MAPPINGS_TEST_ITEM;
+        if (randomBoolean()) {
+            Map stringObjectMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, FIND_MAPPINGS_TEST_ITEM, false);
+            Map doc = (Map)stringObjectMap.get("doc");
+            try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+                builder.map(doc);
+                mapping = builder.string();
+            }
+        }
+
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("index1")
                         .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                         .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
-                .putMapping("doc", FIND_MAPPINGS_TEST_ITEM))
+                .putMapping("doc", mapping))
                 .put(IndexMetaData.builder("index2")
                         .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
-                        .putMapping("doc", FIND_MAPPINGS_TEST_ITEM))
+                        .putMapping("doc", mapping))
                 .put(IndexMetaData.builder("index3")
                         .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
-                        .putMapping("doc", FIND_MAPPINGS_TEST_ITEM)).build();
+                        .putMapping("doc", mapping)).build();
 
         {
             ImmutableOpenMap> mappings = metaData.findMappings(
diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
index f65250a5666..a58e9abbd47 100644
--- a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
@@ -36,11 +36,9 @@ import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.hamcrest.Matchers;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -107,7 +105,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
         }
     }
 
-    public void testUpdateMappingWithoutType() throws Exception {
+    public void testUpdateMappingWithoutType() {
         client().admin().indices().prepareCreate("test")
                 .setSettings(
                         Settings.builder()
@@ -128,7 +126,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
                 equalTo("{\"doc\":{\"properties\":{\"body\":{\"type\":\"text\"},\"date\":{\"type\":\"integer\"}}}}"));
     }
 
-    public void testUpdateMappingWithoutTypeMultiObjects() throws Exception {
+    public void testUpdateMappingWithoutTypeMultiObjects() {
         client().admin().indices().prepareCreate("test")
                 .setSettings(
                         Settings.builder()
@@ -148,7 +146,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
                 equalTo("{\"doc\":{\"properties\":{\"date\":{\"type\":\"integer\"}}}}"));
     }
 
-    public void testUpdateMappingWithConflicts() throws Exception {
+    public void testUpdateMappingWithConflicts() {
         client().admin().indices().prepareCreate("test")
                 .setSettings(
                         Settings.builder()
@@ -167,7 +165,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
         }
     }
 
-    public void testUpdateMappingWithNormsConflicts() throws Exception {
+    public void testUpdateMappingWithNormsConflicts() {
         client().admin().indices().prepareCreate("test")
                 .addMapping("type", "{\"type\":{\"properties\":{\"body\":{\"type\":\"text\", \"norms\": false }}}}", XContentType.JSON)
                 .execute().actionGet();
@@ -184,7 +182,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
     /*
     Second regression test for https://github.com/elastic/elasticsearch/issues/3381
      */
-    public void testUpdateMappingNoChanges() throws Exception {
+    public void testUpdateMappingNoChanges() {
         client().admin().indices().prepareCreate("test")
                 .setSettings(
                         Settings.builder()
@@ -251,7 +249,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
         getResponse = client().admin().indices().prepareGetMappings("test").addTypes(MapperService.DEFAULT_MAPPING).get();
         defaultMapping = getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap();
         Map fieldSettings = (Map) ((Map) defaultMapping.get("properties")).get("f");
-        assertThat(fieldSettings, hasEntry("type", (Object) "keyword"));
+        assertThat(fieldSettings, hasEntry("type", "keyword"));
 
         // but we still validate the _default_ type
         logger.info("Confirming _default_ mappings validation");
@@ -276,40 +274,36 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
         }
 
         for (int j = 0; j < threads.length; j++) {
-            threads[j] = new Thread(new Runnable() {
-                @SuppressWarnings("unchecked")
-                @Override
-                public void run() {
-                    try {
-                        barrier.await();
+            threads[j] = new Thread(() -> {
+                try {
+                    barrier.await();
 
-                        for (int i = 0; i < 100; i++) {
-                            if (stop.get()) {
-                                return;
-                            }
-
-                            Client client1 = clientArray.get(i % clientArray.size());
-                            Client client2 = clientArray.get((i + 1) % clientArray.size());
-                            String indexName = i % 2 == 0 ? "test2" : "test1";
-                            String typeName = "type";
-                            String fieldName = Thread.currentThread().getName() + "_" + i;
-
-                            PutMappingResponse response = client1.admin().indices().preparePutMapping(indexName).setType(typeName).setSource(
-                                    JsonXContent.contentBuilder().startObject().startObject(typeName)
-                                            .startObject("properties").startObject(fieldName).field("type", "text").endObject().endObject()
-                                            .endObject().endObject()
-                            ).get();
-
-                            assertThat(response.isAcknowledged(), equalTo(true));
-                            GetMappingsResponse getMappingResponse = client2.admin().indices().prepareGetMappings(indexName).get();
-                            ImmutableOpenMap mappings = getMappingResponse.getMappings().get(indexName);
-                            assertThat(mappings.containsKey(typeName), equalTo(true));
-                            assertThat(((Map) mappings.get(typeName).getSourceAsMap().get("properties")).keySet(), Matchers.hasItem(fieldName));
+                    for (int i = 0; i < 100; i++) {
+                        if (stop.get()) {
+                            return;
                         }
-                    } catch (Exception e) {
-                        threadException.set(e);
-                        stop.set(true);
+
+                        Client client1 = clientArray.get(i % clientArray.size());
+                        Client client2 = clientArray.get((i + 1) % clientArray.size());
+                        String indexName = i % 2 == 0 ? "test2" : "test1";
+                        String typeName = "type";
+                        String fieldName = Thread.currentThread().getName() + "_" + i;
+
+                        PutMappingResponse response = client1.admin().indices().preparePutMapping(indexName).setType(typeName).setSource(
+                                JsonXContent.contentBuilder().startObject().startObject(typeName)
+                                        .startObject("properties").startObject(fieldName).field("type", "text").endObject().endObject()
+                                        .endObject().endObject()
+                        ).get();
+
+                        assertThat(response.isAcknowledged(), equalTo(true));
+                        GetMappingsResponse getMappingResponse = client2.admin().indices().prepareGetMappings(indexName).get();
+                        ImmutableOpenMap mappings = getMappingResponse.getMappings().get(indexName);
+                        assertThat(mappings.containsKey(typeName), equalTo(true));
+                        assertThat(((Map) mappings.get(typeName).getSourceAsMap().get("properties")).keySet(), Matchers.hasItem(fieldName));
                     }
+                } catch (Exception e) {
+                    threadException.set(e);
+                    stop.set(true);
                 }
             });
 
@@ -325,7 +319,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
 
     }
 
-    public void testPutMappingsWithBlocks() throws Exception {
+    public void testPutMappingsWithBlocks() {
         createIndex("test");
         ensureGreen();
 
@@ -350,7 +344,8 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
         }
     }
 
-    public void testUpdateMappingOnAllTypes() throws IOException {
+    @SuppressWarnings("unchecked")
+    public void testUpdateMappingOnAllTypes() {
         assertTrue("remove this multi type test", Version.CURRENT.before(Version.fromString("7.0.0")));
         assertAcked(prepareCreate("index")
                 .setSettings(Settings.builder().put("index.version.created", Version.V_5_6_0.id))

From 70f8ea367bf892728b89b08070d6d4804ee2ef5f Mon Sep 17 00:00:00 2001
From: Simon Willnauer 
Date: Wed, 6 Dec 2017 07:35:37 +0100
Subject: [PATCH 195/297] Allow index settings to be reset by wildcards
 (#27671)

Index settings didn't support reset by wildcard which also causes
issues like #27537 where archived settings can't be reset. This change
adds support for wildcards like `archived.*` to be used to reset setting to their
defaults or remove them from an index.

Closes #27537
---
 .../MetaDataUpdateSettingsService.java        | 13 ++++---
 .../settings/AbstractScopedSettings.java      | 23 +++++++-----
 .../common/settings/Settings.java             | 12 +++++-
 .../indices/settings/UpdateSettingsIT.java    | 37 ++++++++++++++++++-
 4 files changed, 68 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
index 2c0bc929294..59c38be50e8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
@@ -38,6 +38,7 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -54,7 +55,6 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
-import java.util.function.Predicate;
 
 import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
 
@@ -164,13 +164,16 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
         Settings.Builder settingsForOpenIndices = Settings.builder();
         final Set skippedSettings = new HashSet<>();
 
-        indexScopedSettings.validate(normalizedSettings, false); // don't validate dependencies here we check it below
-        // never allow to change the number of shards
+        indexScopedSettings.validate(normalizedSettings.filter(s -> Regex.isSimpleMatchPattern(s) == false  /* don't validate wildcards */),
+            false); //don't validate dependencies here we check it below never allow to change the number of shards
         for (String key : normalizedSettings.keySet()) {
             Setting setting = indexScopedSettings.get(key);
-            assert setting != null; // we already validated the normalized settings
+            boolean isWildcard = setting == null && Regex.isSimpleMatchPattern(key);
+            assert setting != null // we already validated the normalized settings
+                || (isWildcard && normalizedSettings.hasValue(key) == false)
+                : "unknown setting: " + key + " isWildcard: " + isWildcard + " hasValue: " + normalizedSettings.hasValue(key);
             settingsForClosedIndices.copy(key, normalizedSettings);
-            if (setting.isDynamic()) {
+            if (isWildcard || setting.isDynamic()) {
                 settingsForOpenIndices.copy(key, normalizedSettings);
             } else {
                 skippedSettings.add(key);
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index 38eaef1d14d..f952eb36a0d 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -500,6 +500,16 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
         return updateSettings(toApply, target, updates, type, false);
     }
 
+    /**
+     * Returns true if the given key is a valid delete key
+     */
+    private boolean isValidDelete(String key, boolean onlyDynamic) {
+        return isFinalSetting(key) == false && // it's not a final setting
+            (onlyDynamic && isDynamicSetting(key)  // it's a dynamicSetting and we only do dynamic settings
+                || get(key) == null && key.startsWith(ARCHIVED_SETTINGS_PREFIX) // the setting is not registered AND it's been archived
+                || (onlyDynamic == false && get(key) != null)); // if it's not dynamic AND we have a key
+    }
+
     /**
      * Updates a target settings builder with new, updated or deleted settings from a given settings builder.
      *
@@ -519,21 +529,16 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
         final Predicate canUpdate = (key) -> (
             isFinalSetting(key) == false && // it's not a final setting
                 ((onlyDynamic == false && get(key) != null) || isDynamicSetting(key)));
-        final Predicate canRemove = (key) ->(// we can delete if
-            isFinalSetting(key) == false && // it's not a final setting
-                (onlyDynamic && isDynamicSetting(key)  // it's a dynamicSetting and we only do dynamic settings
-                || get(key) == null && key.startsWith(ARCHIVED_SETTINGS_PREFIX) // the setting is not registered AND it's been archived
-                || (onlyDynamic == false && get(key) != null))); // if it's not dynamic AND we have a key
         for (String key : toApply.keySet()) {
-            boolean isNull = toApply.get(key) == null;
-            if (isNull && (canRemove.test(key) || key.endsWith("*"))) {
+            boolean isDelete = toApply.hasValue(key) == false;
+            if (isDelete && (isValidDelete(key, onlyDynamic) || key.endsWith("*"))) {
                 // this either accepts null values that suffice the canUpdate test OR wildcard expressions (key ends with *)
                 // we don't validate if there is any dynamic setting with that prefix yet we could do in the future
                 toRemove.add(key);
                 // we don't set changed here it's set after we apply deletes below if something actually changed
             } else if (get(key) == null) {
                 throw new IllegalArgumentException(type + " setting [" + key + "], not recognized");
-            } else if (isNull == false && canUpdate.test(key)) {
+            } else if (isDelete == false && canUpdate.test(key)) {
                 validate(key, toApply, false); // we might not have a full picture here do to a dependency validation
                 settingsBuilder.copy(key, toApply);
                 updates.copy(key, toApply);
@@ -546,7 +551,7 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
                 }
             }
         }
-        changed |= applyDeletes(toRemove, target, canRemove);
+        changed |= applyDeletes(toRemove, target, k -> isValidDelete(k, onlyDynamic));
         target.put(settingsBuilder.build());
         return changed;
     }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
index 41acefdd8e8..3648abb78c0 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -306,6 +306,13 @@ public final class Settings implements ToXContentFragment {
         }
     }
 
+    /**
+     * Returns true iff the given key has a value in this settings object
+     */
+    public boolean hasValue(String key) {
+        return settings.get(key) != null;
+    }
+
     /**
      * We have to lazy initialize the deprecation logger as otherwise a static logger here would be constructed before logging is configured
      * leading to a runtime failure (see {@link LogConfigurator#checkErrorListener()} ). The premature construction would come from any
@@ -1229,8 +1236,9 @@ public final class Settings implements ToXContentFragment {
             Iterator> iterator = map.entrySet().iterator();
             while(iterator.hasNext()) {
                 Map.Entry entry = iterator.next();
-                if (entry.getKey().startsWith(prefix) == false) {
-                    replacements.put(prefix + entry.getKey(), entry.getValue());
+                String key = entry.getKey();
+                if (key.startsWith(prefix) == false && key.endsWith("*") == false) {
+                    replacements.put(prefix + key, entry.getValue());
                     iterator.remove();
                 }
             }
diff --git a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
index 7ff0725449e..51c073c607e 100644
--- a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
@@ -241,10 +241,45 @@ public class UpdateSettingsIT extends ESIntegTestCase {
                 .actionGet();
         }
     }
+    public void testResetDefaultWithWildcard() {
+        createIndex("test");
+
+        client()
+            .admin()
+            .indices()
+            .prepareUpdateSettings("test")
+            .setSettings(
+                Settings.builder()
+                    .put("index.refresh_interval", -1))
+            .execute()
+            .actionGet();
+        IndexMetaData indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
+        assertEquals(indexMetaData.getSettings().get("index.refresh_interval"), "-1");
+        for (IndicesService service : internalCluster().getInstances(IndicesService.class)) {
+            IndexService indexService = service.indexService(resolveIndex("test"));
+            if (indexService != null) {
+                assertEquals(indexService.getIndexSettings().getRefreshInterval().millis(), -1);
+            }
+        }
+        client()
+            .admin()
+            .indices()
+            .prepareUpdateSettings("test")
+            .setSettings(Settings.builder().putNull("index.ref*"))
+            .execute()
+            .actionGet();
+        indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
+        assertNull(indexMetaData.getSettings().get("index.refresh_interval"));
+        for (IndicesService service : internalCluster().getInstances(IndicesService.class)) {
+            IndexService indexService = service.indexService(resolveIndex("test"));
+            if (indexService != null) {
+                assertEquals(indexService.getIndexSettings().getRefreshInterval().millis(), 1000);
+            }
+        }
+    }
 
     public void testResetDefault() {
         createIndex("test");
-
         client()
             .admin()
             .indices()

From caea6b70faf259ed1b0da01ff4c75d936438bb16 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi 
Date: Wed, 6 Dec 2017 09:15:28 +0100
Subject: [PATCH 196/297] Add a new cluster setting to limit the total number
 of buckets returned by a request (#27581)

This commit adds a new dynamic cluster setting named `search.max_buckets` that can be used to limit the number of buckets created per shard or by the reduce phase. Each multi bucket aggregator can consume buckets during the final build of the aggregation at the shard level or during the reduce phase (final or not) in the coordinating node. When an aggregator consumes a bucket, a global count for the request is incremented and if this number is greater than the limit an exception is thrown (TooManyBuckets exception).
This change adds the ability for multi bucket aggregator to "consume" buckets in the global limit, the default is 10,000. It's an opt-in consumer so each multi-bucket aggregator must explicitly call the consumer when a bucket is added in the response.

Closes #27452 #26012
---
 .../elasticsearch/ElasticsearchException.java |   6 +-
 .../action/search/SearchPhaseController.java  |  18 ++-
 .../common/settings/ClusterSettings.java      |   2 +
 .../java/org/elasticsearch/node/Node.java     |  14 +-
 .../elasticsearch/search/SearchService.java   |  12 +-
 .../search/aggregations/AggregationPhase.java |   1 +
 .../aggregations/InternalAggregation.java     |  16 +++
 .../InternalMultiBucketAggregation.java       |  34 +++++
 .../MultiBucketConsumerService.java           | 126 ++++++++++++++++++
 .../SearchContextAggregations.java            |  19 ++-
 .../bucket/BucketsAggregator.java             |  15 +++
 .../adjacency/AdjacencyMatrixAggregator.java  |   2 +
 .../adjacency/InternalAdjacencyMatrix.java    |   3 +
 .../bucket/composite/CompositeAggregator.java |   1 +
 .../bucket/composite/InternalComposite.java   |   2 +
 .../bucket/filter/FiltersAggregator.java      |   1 +
 .../bucket/filter/InternalFilters.java        |   3 +-
 .../bucket/geogrid/GeoHashGridAggregator.java |   1 +
 .../bucket/geogrid/InternalGeoHashGrid.java   |   7 +-
 .../histogram/DateHistogramAggregator.java    |   2 +
 .../bucket/histogram/HistogramAggregator.java |   1 +
 .../histogram/InternalDateHistogram.java      |  10 ++
 .../bucket/histogram/InternalHistogram.java   |  10 ++
 .../bucket/range/BinaryRangeAggregator.java   |   1 +
 .../bucket/range/InternalBinaryRange.java     |   1 +
 .../bucket/range/InternalRange.java           |   1 +
 .../bucket/range/RangeAggregator.java         |   1 +
 ...balOrdinalsSignificantTermsAggregator.java |   3 +
 .../significant/InternalSignificantTerms.java |   9 +-
 .../SignificantLongTermsAggregator.java       |   3 +
 .../SignificantStringTermsAggregator.java     |   3 +
 .../SignificantTextAggregator.java            |  39 +++---
 .../GlobalOrdinalsStringTermsAggregator.java  |   1 +
 .../bucket/terms/InternalTerms.java           |   5 +
 .../bucket/terms/LongTermsAggregator.java     |   6 +-
 .../bucket/terms/StringTermsAggregator.java   |   3 +
 .../ExceptionSerializationTests.java          |  10 ++
 .../action/search/DfsQueryPhaseTests.java     |  10 +-
 .../action/search/FetchSearchPhaseTests.java  |  19 ++-
 .../search/SearchPhaseControllerTests.java    |   4 +-
 .../search/aggregations/EquivalenceIT.java    |  17 +++
 .../composite/CompositeAggregatorTests.java   |   2 +-
 .../DateHistogramAggregatorTests.java         |  71 +++++++++-
 .../bucket/terms/TermsAggregatorTests.java    |   5 +-
 docs/reference/aggregations/bucket.asciidoc   |   4 +
 .../migrate_7_0/aggregations.asciidoc         |   8 +-
 .../search.aggregation/240_max_buckets.yml    | 110 +++++++++++++++
 .../aggregations/AggregatorTestCase.java      |  83 +++++++++---
 .../test/InternalAggregationTestCase.java     |  17 ++-
 49 files changed, 658 insertions(+), 84 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/MultiBucketConsumerService.java
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml

diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
index f4e807b9ffc..9e4e7b909f7 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -34,6 +34,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.transport.TcpTransport;
 
 import java.io.IOException;
@@ -986,7 +987,10 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         SHARD_LOCK_OBTAIN_FAILED_EXCEPTION(org.elasticsearch.env.ShardLockObtainFailedException.class,
                                            org.elasticsearch.env.ShardLockObtainFailedException::new, 147, Version.V_5_0_2),
         UNKNOWN_NAMED_OBJECT_EXCEPTION(org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException.class,
-                org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException::new, 148, Version.V_5_2_0);
+                org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException::new, 148, Version.V_5_2_0),
+        TOO_MANY_BUCKETS_EXCEPTION(MultiBucketConsumerService.TooManyBucketsException.class,
+                                   MultiBucketConsumerService.TooManyBucketsException::new, 149,
+            Version.V_7_0_0_alpha1);
 
         final Class exceptionClass;
         final CheckedFunction constructor;
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index 6525ad021d0..ba546660f93 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -65,6 +65,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.IntFunction;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
@@ -73,13 +74,16 @@ public final class SearchPhaseController extends AbstractComponent {
 
     private static final ScoreDoc[] EMPTY_DOCS = new ScoreDoc[0];
 
-    private final BigArrays bigArrays;
-    private final ScriptService scriptService;
+    private final Function reduceContextFunction;
 
-    public SearchPhaseController(Settings settings, BigArrays bigArrays, ScriptService scriptService) {
+    /**
+     * Constructor.
+     * @param settings Node settings
+     * @param reduceContextFunction A function that builds a context for the reduce of an {@link InternalAggregation}
+     */
+    public SearchPhaseController(Settings settings, Function reduceContextFunction) {
         super(settings);
-        this.bigArrays = bigArrays;
-        this.scriptService = scriptService;
+        this.reduceContextFunction = reduceContextFunction;
     }
 
     public AggregatedDfs aggregateDfs(Collection results) {
@@ -496,7 +500,7 @@ public final class SearchPhaseController extends AbstractComponent {
             }
         }
         final Suggest suggest = groupedSuggestions.isEmpty() ? null : new Suggest(Suggest.reduce(groupedSuggestions));
-        ReduceContext reduceContext = new ReduceContext(bigArrays, scriptService, true);
+        ReduceContext reduceContext = reduceContextFunction.apply(true);
         final InternalAggregations aggregations = aggregationsList.isEmpty() ? null : reduceAggs(aggregationsList,
             firstResult.pipelineAggregators(), reduceContext);
         final SearchProfileShardResults shardResults = profileResults.isEmpty() ? null : new SearchProfileShardResults(profileResults);
@@ -513,7 +517,7 @@ public final class SearchPhaseController extends AbstractComponent {
      * that relevant for the final reduce step. For final reduce see {@link #reduceAggs(List, List, ReduceContext)}
      */
     private InternalAggregations reduceAggsIncrementally(List aggregationsList) {
-        ReduceContext reduceContext = new ReduceContext(bigArrays, scriptService, false);
+        ReduceContext reduceContext = reduceContextFunction.apply(false);
         return aggregationsList.isEmpty() ? null : reduceAggs(aggregationsList,
             null, reduceContext);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 2bea2a59e16..ae28b42cf16 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -85,6 +85,7 @@ import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchService;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteClusterAware;
@@ -360,6 +361,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     SearchService.DEFAULT_KEEPALIVE_SETTING,
                     SearchService.KEEPALIVE_INTERVAL_SETTING,
                     SearchService.MAX_KEEPALIVE_SETTING,
+                    MultiBucketConsumerService.MAX_BUCKET_SETTING,
                     SearchService.LOW_LEVEL_CANCELLATION_SETTING,
                     Node.WRITE_PORTS_FILE_SETTING,
                     Node.NODE_NAME_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index fee6d76ca3d..a4b7e5147d5 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -100,7 +100,6 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
-import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.RecoverySettings;
@@ -449,6 +448,11 @@ public class Node implements Closeable {
                 transportService, indicesService, pluginsService, circuitBreakerService, scriptModule.getScriptService(),
                 httpServerTransport, ingestService, clusterService, settingsModule.getSettingsFilter(), responseCollectorService,
                 searchTransportService);
+
+            final SearchService searchService = newSearchService(clusterService, indicesService,
+                threadPool, scriptModule.getScriptService(), bigArrays, searchModule.getFetchPhase(),
+                responseCollectorService);
+
             modules.add(b -> {
                     b.bind(Node.class).toInstance(this);
                     b.bind(NodeService.class).toInstance(nodeService);
@@ -470,12 +474,10 @@ public class Node implements Closeable {
                     b.bind(MetaDataUpgrader.class).toInstance(metaDataUpgrader);
                     b.bind(MetaStateService.class).toInstance(metaStateService);
                     b.bind(IndicesService.class).toInstance(indicesService);
-                    b.bind(SearchService.class).toInstance(newSearchService(clusterService, indicesService,
-                        threadPool, scriptModule.getScriptService(), bigArrays, searchModule.getFetchPhase(),
-                        responseCollectorService));
+                    b.bind(SearchService.class).toInstance(searchService);
                     b.bind(SearchTransportService.class).toInstance(searchTransportService);
-                    b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController(settings, bigArrays,
-                            scriptModule.getScriptService()));
+                    b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController(settings,
+                        searchService::createReduceContext));
                     b.bind(Transport.class).toInstance(transport);
                     b.bind(TransportService.class).toInstance(transportService);
                     b.bind(NetworkService.class).toInstance(networkService);
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 117a979639b..9ac83276000 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -60,6 +60,8 @@ import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.collapse.CollapseContext;
@@ -118,6 +120,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         Setting.positiveTimeSetting("search.max_keep_alive", timeValueHours(24), Property.NodeScope, Property.Dynamic);
     public static final Setting KEEPALIVE_INTERVAL_SETTING =
         Setting.positiveTimeSetting("search.keep_alive_interval", timeValueMinutes(1), Property.NodeScope);
+
     /**
      * Enables low-level, frequent search cancellation checks. Enabling low-level checks will make long running searches to react
      * to the cancellation request faster. However, since it will produce more cancellation checks it might slow the search performance
@@ -163,6 +166,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
 
     private final ConcurrentMapLong activeContexts = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
 
+    private final MultiBucketConsumerService multiBucketConsumerService;
+
     public SearchService(ClusterService clusterService, IndicesService indicesService,
                          ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays, FetchPhase fetchPhase,
                          ResponseCollectorService responseCollectorService) {
@@ -175,6 +180,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         this.bigArrays = bigArrays;
         this.queryPhase = new QueryPhase(settings);
         this.fetchPhase = fetchPhase;
+        this.multiBucketConsumerService = new MultiBucketConsumerService(clusterService, settings);
 
         TimeValue keepAliveInterval = KEEPALIVE_INTERVAL_SETTING.get(settings);
         setKeepAlives(DEFAULT_KEEPALIVE_SETTING.get(settings), MAX_KEEPALIVE_SETTING.get(settings));
@@ -741,7 +747,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         if (source.aggregations() != null) {
             try {
                 AggregatorFactories factories = source.aggregations().build(context, null);
-                context.aggregations(new SearchContextAggregations(factories));
+                context.aggregations(new SearchContextAggregations(factories, multiBucketConsumerService.create()));
             } catch (IOException e) {
                 throw new AggregationInitializationException("Failed to create aggregators", e);
             }
@@ -1017,4 +1023,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
     public IndicesService getIndicesService() {
         return indicesService;
     }
+
+    public InternalAggregation.ReduceContext createReduceContext(boolean finalReduce) {
+        return new InternalAggregation.ReduceContext(bigArrays, scriptService, multiBucketConsumerService.create(), finalReduce);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
index 9b4a02cd816..4dc765d0db1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
@@ -123,6 +123,7 @@ public class AggregationPhase implements SearchPhase {
         }
 
         List aggregations = new ArrayList<>(aggregators.length);
+        context.aggregations().resetBucketMultiConsumer();
         for (Aggregator aggregator : context.aggregations().aggregators()) {
             try {
                 aggregator.postCollection();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index 4c886d3a9be..7f6e74e68b2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -22,6 +22,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.BigArray;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.action.search.RestSearchAction;
@@ -33,6 +34,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.IntConsumer;
 
 /**
  * An internal implementation of {@link Aggregation}. Serves as a base class for all aggregation implementations.
@@ -43,11 +45,17 @@ public abstract class InternalAggregation implements Aggregation, NamedWriteable
 
         private final BigArrays bigArrays;
         private final ScriptService scriptService;
+        private final IntConsumer multiBucketConsumer;
         private final boolean isFinalReduce;
 
         public ReduceContext(BigArrays bigArrays, ScriptService scriptService, boolean isFinalReduce) {
+            this(bigArrays, scriptService, (s) -> {}, isFinalReduce);
+        }
+
+        public ReduceContext(BigArrays bigArrays, ScriptService scriptService, IntConsumer multiBucketConsumer, boolean isFinalReduce) {
             this.bigArrays = bigArrays;
             this.scriptService = scriptService;
+            this.multiBucketConsumer = multiBucketConsumer;
             this.isFinalReduce = isFinalReduce;
         }
 
@@ -67,6 +75,14 @@ public abstract class InternalAggregation implements Aggregation, NamedWriteable
         public ScriptService scriptService() {
             return scriptService;
         }
+
+        /**
+         * Adds count buckets to the global count for the request and fails if this number is greater than
+         * the maximum number of buckets allowed in a response
+         */
+        public void consumeBucketsAndMaybeBreak(int size) {
+            multiBucketConsumer.accept(size);
+        }
     }
 
     protected final String name;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
index 8e8f4edcf31..9084f415d77 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
@@ -22,6 +22,7 @@ package org.elasticsearch.search.aggregations;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
@@ -82,6 +83,39 @@ public abstract class InternalMultiBucketAggregation path) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/MultiBucketConsumerService.java b/core/src/main/java/org/elasticsearch/search/aggregations/MultiBucketConsumerService.java
new file mode 100644
index 00000000000..63ba70c6f23
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/MultiBucketConsumerService.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
+
+import java.io.IOException;
+import java.util.function.IntConsumer;
+
+/**
+ * An aggregation service that creates instances of {@link MultiBucketConsumer}.
+ * The consumer is used by {@link BucketsAggregator} and {@link InternalMultiBucketAggregation} to limit the number of buckets created
+ * in {@link Aggregator#buildAggregation} and {@link InternalAggregation#reduce}.
+ * The limit can be set by changing the `search.max_buckets` cluster setting and defaults to 10000.
+ */
+public class MultiBucketConsumerService {
+    public static final int DEFAULT_MAX_BUCKETS = 10000;
+    public static final Setting MAX_BUCKET_SETTING =
+        Setting.intSetting("search.max_buckets", DEFAULT_MAX_BUCKETS, 0, Setting.Property.NodeScope, Setting.Property.Dynamic);
+
+    private volatile int maxBucket;
+
+    public MultiBucketConsumerService(ClusterService clusterService, Settings settings) {
+       this.maxBucket = MAX_BUCKET_SETTING.get(settings);
+       clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_BUCKET_SETTING, this::setMaxBucket);
+    }
+
+    private void setMaxBucket(int maxBucket) {
+        this.maxBucket = maxBucket;
+    }
+
+    public static class TooManyBucketsException extends AggregationExecutionException {
+        private final int maxBuckets;
+
+        public TooManyBucketsException(String message, int maxBuckets) {
+            super(message);
+            this.maxBuckets = maxBuckets;
+        }
+
+        public TooManyBucketsException(StreamInput in) throws IOException {
+            super(in);
+            maxBuckets = in.readInt();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeInt(maxBuckets);
+        }
+
+        public int getMaxBuckets() {
+            return maxBuckets;
+        }
+
+        @Override
+        public RestStatus status() {
+            return RestStatus.SERVICE_UNAVAILABLE;
+        }
+
+        @Override
+        protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field("max_buckets", maxBuckets);
+        }
+    }
+
+    /**
+     * An {@link IntConsumer} that throws a {@link TooManyBucketsException}
+     * when the sum of the provided values is above the limit (`search.max_buckets`).
+     * It is used by aggregators to limit the number of bucket creation during
+     * {@link Aggregator#buildAggregation} and {@link InternalAggregation#reduce}.
+     */
+    public static class MultiBucketConsumer implements IntConsumer {
+        private final int limit;
+        // aggregations execute in a single thread so no atomic here
+        private int count;
+
+        public MultiBucketConsumer(int limit) {
+            this.limit = limit;
+        }
+
+        @Override
+        public void accept(int value) {
+            count += value;
+            if (count > limit) {
+                throw new TooManyBucketsException("Trying to create too many buckets. Must be less than or equal to: [" + limit
+                    + "] but was [" + count + "]. This limit can be set by changing the [" +
+                    MAX_BUCKET_SETTING.getKey() + "] cluster level setting.", limit);
+            }
+        }
+
+        public void reset() {
+            this.count = 0;
+        }
+
+        public int getCount() {
+            return count;
+        }
+    }
+
+    public MultiBucketConsumer create() {
+        return new MultiBucketConsumer(maxBucket);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java b/core/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java
index 9476af03846..ab0a73d53ed 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/SearchContextAggregations.java
@@ -18,19 +18,25 @@
  */
 package org.elasticsearch.search.aggregations;
 
+import java.util.function.IntConsumer;
+
+import static org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer;
+
 /**
  * The aggregation context that is part of the search context.
  */
 public class SearchContextAggregations {
 
     private final AggregatorFactories factories;
+    private final MultiBucketConsumer multiBucketConsumer;
     private Aggregator[] aggregators;
 
     /**
      * Creates a new aggregation context with the parsed aggregator factories
      */
-    public SearchContextAggregations(AggregatorFactories factories) {
+    public SearchContextAggregations(AggregatorFactories factories, MultiBucketConsumer multiBucketConsumer) {
         this.factories = factories;
+        this.multiBucketConsumer = multiBucketConsumer;
     }
 
     public AggregatorFactories factories() {
@@ -50,4 +56,15 @@ public class SearchContextAggregations {
         this.aggregators = aggregators;
     }
 
+    /**
+     * Returns a consumer for multi bucket aggregation that checks the total number of buckets
+     * created in the response
+     */
+    public IntConsumer multiBucketConsumer() {
+        return multiBucketConsumer;
+    }
+
+    void resetBucketMultiConsumer() {
+        multiBucketConsumer.reset();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
index 546cb07af26..02cf3adf88a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
@@ -34,10 +34,12 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.function.IntConsumer;
 
 public abstract class BucketsAggregator extends AggregatorBase {
 
     private final BigArrays bigArrays;
+    private final IntConsumer multiBucketConsumer;
     private IntArray docCounts;
 
     public BucketsAggregator(String name, AggregatorFactories factories, SearchContext context, Aggregator parent,
@@ -45,6 +47,11 @@ public abstract class BucketsAggregator extends AggregatorBase {
         super(name, factories, context, parent, pipelineAggregators, metaData);
         bigArrays = context.bigArrays();
         docCounts = bigArrays.newIntArray(1, true);
+        if (context.aggregations() != null) {
+            multiBucketConsumer = context.aggregations().multiBucketConsumer();
+        } else {
+            multiBucketConsumer = (count) -> {};
+        }
     }
 
     /**
@@ -104,6 +111,14 @@ public abstract class BucketsAggregator extends AggregatorBase {
         }
     }
 
+    /**
+     * Adds count buckets to the global count for the request and fails if this number is greater than
+     * the maximum number of buckets allowed in a response
+     */
+    protected final void consumeBucketsAndMaybeBreak(int count) {
+        multiBucketConsumer.accept(count);
+    }
+
     /**
      * Required method to build the child aggregations of the given bucket (identified by the bucket ordinal).
      */
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java
index dd1e89897af..71314d1f539 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java
@@ -210,6 +210,7 @@ public class AdjacencyMatrixAggregator extends BucketsAggregator {
                 InternalAdjacencyMatrix.InternalBucket bucket = new InternalAdjacencyMatrix.InternalBucket(keys[i],
                         docCount, bucketAggregations(bucketOrd));
                 buckets.add(bucket);
+                consumeBucketsAndMaybeBreak(1);
             }
         }
         int pos = keys.length;
@@ -223,6 +224,7 @@ public class AdjacencyMatrixAggregator extends BucketsAggregator {
                     InternalAdjacencyMatrix.InternalBucket bucket = new InternalAdjacencyMatrix.InternalBucket(intersectKey,
                             docCount, bucketAggregations(bucketOrd));
                     buckets.add(bucket);
+                    consumeBucketsAndMaybeBreak(1);
                 }
                 pos++;
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
index 602a0964ee9..8ce6304daf8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
@@ -214,7 +214,10 @@ public class InternalAdjacencyMatrix
         for (List sameRangeList : bucketsMap.values()) {
             InternalBucket reducedBucket = sameRangeList.get(0).reduce(sameRangeList, reduceContext);
             if(reducedBucket.docCount >= 1){
+                reduceContext.consumeBucketsAndMaybeBreak(1);
                 reducedBuckets.add(reducedBucket);
+            } else {
+                reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(reducedBucket));
             }
         }
         Collections.sort(reducedBuckets, Comparator.comparing(InternalBucket::getKey));
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
index c18c414abb6..9612ba2f895 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
@@ -83,6 +83,7 @@ final class CompositeAggregator extends BucketsAggregator {
     @Override
     public InternalAggregation buildAggregation(long zeroBucket) throws IOException {
         assert zeroBucket == 0L;
+        consumeBucketsAndMaybeBreak(keys.size());
 
         // Replay all documents that contain at least one top bucket (collected during the first pass).
         grow(keys.size()+1);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
index bfeabcb9833..824250948d7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
@@ -132,6 +132,7 @@ public class InternalComposite
             if (lastBucket != null && bucketIt.current.compareKey(lastBucket) != 0) {
                 InternalBucket reduceBucket = buckets.get(0).reduce(buckets, reduceContext);
                 buckets.clear();
+                reduceContext.consumeBucketsAndMaybeBreak(1);
                 result.add(reduceBucket);
                 if (result.size() >= size) {
                     break;
@@ -145,6 +146,7 @@ public class InternalComposite
         }
         if (buckets.size() > 0) {
             InternalBucket reduceBucket = buckets.get(0).reduce(buckets, reduceContext);
+            reduceContext.consumeBucketsAndMaybeBreak(1);
             result.add(reduceBucket);
         }
         return new InternalComposite(name, size, sourceNames, result, reverseMuls, pipelineAggregators(), metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java
index 97724aa8b97..80d5164a96c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregator.java
@@ -166,6 +166,7 @@ public class FiltersAggregator extends BucketsAggregator {
 
     @Override
     public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
+        consumeBucketsAndMaybeBreak(keys.length + (showOtherBucket ? 1 : 0));
         List buckets = new ArrayList<>(keys.length);
         for (int i = 0; i < keys.length; i++) {
             long bucketOrd = bucketOrd(owningBucketOrdinal, i);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilters.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilters.java
index b43ab7714e4..e522392cf4b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilters.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilters.java
@@ -223,7 +223,8 @@ public class InternalFilters extends InternalMultiBucketAggregation(bucketsList.size()), keyed, pipelineAggregators(),
+        reduceContext.consumeBucketsAndMaybeBreak(bucketsList.size());
+        InternalFilters reduced = new InternalFilters(name, new ArrayList<>(bucketsList.size()), keyed, pipelineAggregators(),
                 getMetaData());
         for (List sameRangeList : bucketsList) {
             reduced.buckets.add((sameRangeList.get(0)).reduce(sameRangeList, reduceContext));
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
index ebcb7b39dba..ec54abb3340 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
@@ -106,6 +106,7 @@ public class GeoHashGridAggregator extends BucketsAggregator {
     public InternalGeoHashGrid buildAggregation(long owningBucketOrdinal) throws IOException {
         assert owningBucketOrdinal == 0;
         final int size = (int) Math.min(bucketOrds.size(), shardSize);
+        consumeBucketsAndMaybeBreak(size);
 
         InternalGeoHashGrid.BucketPriorityQueue ordered = new InternalGeoHashGrid.BucketPriorityQueue(size);
         OrdinalBucket spare = null;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
index 20bccb68305..bc60f5945eb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
@@ -211,7 +211,12 @@ public class InternalGeoHashGrid extends InternalMultiBucketAggregation> cursor : buckets) {
             List sameCellBuckets = cursor.value;
-            ordered.insertWithOverflow(sameCellBuckets.get(0).reduce(sameCellBuckets, reduceContext));
+            Bucket removed = ordered.insertWithOverflow(sameCellBuckets.get(0).reduce(sameCellBuckets, reduceContext));
+            if (removed != null) {
+                reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(removed));
+            } else {
+                reduceContext.consumeBucketsAndMaybeBreak(1);
+            }
         }
         buckets.close();
         Bucket[] list = new Bucket[ordered.size()];
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
index f5f7877572a..8d879b88b3d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
@@ -127,6 +127,8 @@ class DateHistogramAggregator extends BucketsAggregator {
     @Override
     public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
         assert owningBucketOrdinal == 0;
+        consumeBucketsAndMaybeBreak((int) bucketOrds.size());
+
         List buckets = new ArrayList<>((int) bucketOrds.size());
         for (long i = 0; i < bucketOrds.size(); i++) {
             buckets.add(new InternalDateHistogram.Bucket(bucketOrds.get(i), bucketDocCount(i), keyed, formatter, bucketAggregations(i)));
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java
index 0c2ba554c0b..4938daad65b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java
@@ -131,6 +131,7 @@ class HistogramAggregator extends BucketsAggregator {
     @Override
     public InternalAggregation buildAggregation(long bucket) throws IOException {
         assert bucket == 0;
+        consumeBucketsAndMaybeBreak((int) bucketOrds.size());
         List buckets = new ArrayList<>((int) bucketOrds.size());
         for (long i = 0; i < bucketOrds.size(); i++) {
             double roundKey = Double.longBitsToDouble(bucketOrds.get(i));
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
index 1981b313b9c..dfa12db0cd3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
@@ -344,7 +344,10 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation<
                     // the key changes, reduce what we already buffered and reset the buffer for current buckets
                     final Bucket reduced = currentBuckets.get(0).reduce(currentBuckets, reduceContext);
                     if (reduced.getDocCount() >= minDocCount || reduceContext.isFinalReduce() == false) {
+                        reduceContext.consumeBucketsAndMaybeBreak(1);
                         reducedBuckets.add(reduced);
+                    } else {
+                        reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(reduced));
                     }
                     currentBuckets.clear();
                     key = top.current.key;
@@ -365,7 +368,10 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation<
             if (currentBuckets.isEmpty() == false) {
                 final Bucket reduced = currentBuckets.get(0).reduce(currentBuckets, reduceContext);
                 if (reduced.getDocCount() >= minDocCount || reduceContext.isFinalReduce() == false) {
+                    reduceContext.consumeBucketsAndMaybeBreak(1);
                     reducedBuckets.add(reduced);
+                } else {
+                    reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(reduced));
                 }
             }
         }
@@ -388,6 +394,7 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation<
                     long key = bounds.getMin() + offset;
                     long max = bounds.getMax() + offset;
                     while (key <= max) {
+                        reduceContext.consumeBucketsAndMaybeBreak(1);
                         iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
                         key = nextKey(key).longValue();
                     }
@@ -397,6 +404,7 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation<
                     long key = bounds.getMin() + offset;
                     if (key < firstBucket.key) {
                         while (key < firstBucket.key) {
+                            reduceContext.consumeBucketsAndMaybeBreak(1);
                             iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
                             key = nextKey(key).longValue();
                         }
@@ -412,6 +420,7 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation<
             if (lastBucket != null) {
                 long key = nextKey(lastBucket.key).longValue();
                 while (key < nextBucket.key) {
+                    reduceContext.consumeBucketsAndMaybeBreak(1);
                     iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
                     key = nextKey(key).longValue();
                 }
@@ -425,6 +434,7 @@ public final class InternalDateHistogram extends InternalMultiBucketAggregation<
             long key = nextKey(lastBucket.key).longValue();
             long max = bounds.getMax() + offset;
             while (key <= max) {
+                reduceContext.consumeBucketsAndMaybeBreak(1);
                 iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
                 key = nextKey(key).longValue();
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
index aa94bb76259..b3516b04dfc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
@@ -326,7 +326,10 @@ public final class InternalHistogram extends InternalMultiBucketAggregation= minDocCount || reduceContext.isFinalReduce() == false) {
+                        reduceContext.consumeBucketsAndMaybeBreak(1);
                         reducedBuckets.add(reduced);
+                    } else {
+                        reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(reduced));
                     }
                     currentBuckets.clear();
                     key = top.current.key;
@@ -347,7 +350,10 @@ public final class InternalHistogram extends InternalMultiBucketAggregation= minDocCount || reduceContext.isFinalReduce() == false) {
+                    reduceContext.consumeBucketsAndMaybeBreak(1);
                     reducedBuckets.add(reduced);
+                } else {
+                    reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(reduced));
                 }
             }
         }
@@ -374,6 +380,7 @@ public final class InternalHistogram extends InternalMultiBucketAggregation buckets = new ArrayList<>(ranges.length);
         for (int i = 0; i < ranges.length; ++i) {
             long bucketOrd = bucket * ranges.length + i;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
index 3336da08597..e7a3c35231c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
@@ -241,6 +241,7 @@ public final class InternalBinaryRange
 
     @Override
     public InternalAggregation doReduce(List aggregations, ReduceContext reduceContext) {
+        reduceContext.consumeBucketsAndMaybeBreak(buckets.size());
         long[] docCounts = new long[buckets.size()];
         InternalAggregations[][] aggs = new InternalAggregations[buckets.size()][];
         for (int i = 0; i < aggs.length; ++i) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
index f5bb0e25c66..9485d534ab9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
@@ -302,6 +302,7 @@ public class InternalRange aggregations, ReduceContext reduceContext) {
+        reduceContext.consumeBucketsAndMaybeBreak(ranges.size());
         List[] rangeList = new List[ranges.size()];
         for (int i = 0; i < rangeList.length; ++i) {
             rangeList[i] = new ArrayList<>();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
index e502de6210b..2416bf99a11 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
@@ -323,6 +323,7 @@ public class RangeAggregator extends BucketsAggregator {
 
     @Override
     public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
+        consumeBucketsAndMaybeBreak(ranges.length);
         List buckets = new ArrayList<>(ranges.length);
         for (int i = 0; i < ranges.length; i++) {
             Range range = ranges[i];
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
index db620775b61..66b8f8d5b15 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
@@ -131,6 +131,9 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
             // global stats
             spare.updateScore(significanceHeuristic);
             spare = ordered.insertWithOverflow(spare);
+            if (spare == null) {
+                consumeBucketsAndMaybeBreak(1);
+            }
         }
 
         final SignificantStringTerms.Bucket[] list = new SignificantStringTerms.Bucket[ordered.size()];
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
index fe072eb009a..42a3c6a849b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
@@ -241,7 +241,14 @@ public abstract class InternalSignificantTerms 0) && (b.subsetDf >= minDocCount)) || reduceContext.isFinalReduce() == false) {
-                ordered.insertWithOverflow(b);
+                B removed = ordered.insertWithOverflow(b);
+                if (removed == null) {
+                    reduceContext.consumeBucketsAndMaybeBreak(1);
+                } else {
+                    reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(removed));
+                }
+            } else {
+                reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(b));
             }
         }
         B[] list = createBucketsArray(ordered.size());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java
index 93396af9961..235b3f41c08 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java
@@ -101,6 +101,9 @@ public class SignificantLongTermsAggregator extends LongTermsAggregator {
 
             spare.bucketOrd = i;
             spare = ordered.insertWithOverflow(spare);
+            if (spare == null) {
+                consumeBucketsAndMaybeBreak(1);
+            }
         }
 
         final SignificantLongTerms.Bucket[] list = new SignificantLongTerms.Bucket[ordered.size()];
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
index c090d80b039..56258758907 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
@@ -107,6 +107,9 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator {
 
             spare.bucketOrd = i;
             spare = ordered.insertWithOverflow(spare);
+            if (spare == null) {
+                consumeBucketsAndMaybeBreak(1);
+            }
         }
 
         final SignificantStringTerms.Bucket[] list = new SignificantStringTerms.Bucket[ordered.size()];
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregator.java
index 1060bf41488..4dae78aa140 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregator.java
@@ -59,7 +59,7 @@ import java.util.stream.Collectors;
 import static java.util.Collections.emptyList;
 
 public class SignificantTextAggregator extends BucketsAggregator {
-    
+
     private final StringFilter includeExclude;
     protected final BucketCountThresholds bucketCountThresholds;
     protected long numCollectedDocs;
@@ -90,20 +90,20 @@ public class SignificantTextAggregator extends BucketsAggregator {
         this.sourceFieldNames = sourceFieldNames;
         bucketOrds = new BytesRefHash(1, context.bigArrays());
         if(filterDuplicateText){
-            dupSequenceSpotter = new DuplicateByteSequenceSpotter();        
+            dupSequenceSpotter = new DuplicateByteSequenceSpotter();
             lastTrieSize = dupSequenceSpotter.getEstimatedSizeInBytes();
         }
     }
 
-    
-    
+
+
 
     @Override
     public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
             final LeafBucketCollector sub) throws IOException {
         final BytesRefBuilder previous = new BytesRefBuilder();
         return new LeafBucketCollectorBase(sub, null) {
-            
+
             @Override
             public void collect(int doc, long bucket) throws IOException {
                 collectFromSource(doc, bucket, fieldName, sourceFieldNames);
@@ -112,8 +112,8 @@ public class SignificantTextAggregator extends BucketsAggregator {
                     dupSequenceSpotter.startNewSequence();
                 }
             }
-            
-            private void processTokenStream(int doc, long bucket, TokenStream ts, BytesRefHash inDocTerms, String fieldText) 
+
+            private void processTokenStream(int doc, long bucket, TokenStream ts, BytesRefHash inDocTerms, String fieldText)
                     throws IOException{
                 if (dupSequenceSpotter != null) {
                     ts = new DeDuplicatingTokenFilter(ts, dupSequenceSpotter);
@@ -151,35 +151,35 @@ public class SignificantTextAggregator extends BucketsAggregator {
                     ts.close();
                 }
             }
-            
+
             private void collectFromSource(int doc, long bucket, String indexedFieldName, String[] sourceFieldNames) throws IOException {
                 MappedFieldType fieldType = context.getQueryShardContext().fieldMapper(indexedFieldName);
                 if(fieldType == null){
                     throw new IllegalArgumentException("Aggregation [" + name + "] cannot process field ["+indexedFieldName
-                            +"] since it is not present");                    
+                            +"] since it is not present");
                 }
 
                 SourceLookup sourceLookup = context.lookup().source();
                 sourceLookup.setSegmentAndDocument(ctx, doc);
                 BytesRefHash inDocTerms = new BytesRefHash(256, context.bigArrays());
-                
-                try {                
+
+                try {
                     for (String sourceField : sourceFieldNames) {
-                        List textsToHighlight = sourceLookup.extractRawValues(sourceField);    
+                        List textsToHighlight = sourceLookup.extractRawValues(sourceField);
                         textsToHighlight = textsToHighlight.stream().map(obj -> {
                             if (obj instanceof BytesRef) {
                                 return fieldType.valueForDisplay(obj).toString();
                             } else {
                                 return obj;
                             }
-                        }).collect(Collectors.toList());                
-                        
-                        Analyzer analyzer = fieldType.indexAnalyzer();                
+                        }).collect(Collectors.toList());
+
+                        Analyzer analyzer = fieldType.indexAnalyzer();
                         for (Object fieldValue : textsToHighlight) {
                             String fieldText = fieldValue.toString();
                             TokenStream ts = analyzer.tokenStream(indexedFieldName, fieldText);
-                            processTokenStream(doc, bucket, ts, inDocTerms, fieldText);                     
-                        }                    
+                            processTokenStream(doc, bucket, ts, inDocTerms, fieldText);
+                        }
                     }
                 } finally{
                     Releasables.close(inDocTerms);
@@ -220,7 +220,10 @@ public class SignificantTextAggregator extends BucketsAggregator {
             spare.updateScore(significanceHeuristic);
 
             spare.bucketOrd = i;
-            spare = ordered.insertWithOverflow(spare);            
+            spare = ordered.insertWithOverflow(spare);
+            if (spare == null) {
+                consumeBucketsAndMaybeBreak(1);
+            }
         }
 
         final SignificantStringTerms.Bucket[] list = new SignificantStringTerms.Bucket[ordered.size()];
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
index 55023eb263f..6ad14b8d0f9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
@@ -204,6 +204,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
             if (bucketCountThresholds.getShardMinDocCount() <= spare.docCount) {
                 spare = ordered.insertWithOverflow(spare);
                 if (spare == null) {
+                    consumeBucketsAndMaybeBreak(1);
                     spare = new OrdBucket(-1, 0, null, showTermDocCountError, 0);
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
index 529191ac311..72a641ea5bb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
@@ -293,7 +293,12 @@ public abstract class InternalTerms, B extends Int
                 B removed = ordered.insertWithOverflow(b);
                 if (removed != null) {
                     otherDocCount += removed.getDocCount();
+                    reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(removed));
+                } else {
+                    reduceContext.consumeBucketsAndMaybeBreak(1);
                 }
+            } else {
+                reduceContext.consumeBucketsAndMaybeBreak(-countInnerBucket(b));
             }
         }
         B[] list = createBucketsArray(ordered.size());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java
index 4a3190571d7..7cd2c4e9b3a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java
@@ -125,7 +125,6 @@ public class LongTermsAggregator extends TermsAggregator {
         }
 
         final int size = (int) Math.min(bucketOrds.size(), bucketCountThresholds.getShardSize());
-
         long otherDocCount = 0;
         BucketPriorityQueue ordered = new BucketPriorityQueue<>(size, order.comparator(this));
         LongTerms.Bucket spare = null;
@@ -138,7 +137,10 @@ public class LongTermsAggregator extends TermsAggregator {
             otherDocCount += spare.docCount;
             spare.bucketOrd = i;
             if (bucketCountThresholds.getShardMinDocCount() <= spare.docCount) {
-                spare = (LongTerms.Bucket) ordered.insertWithOverflow(spare);
+                spare = ordered.insertWithOverflow(spare);
+                if (spare == null) {
+                    consumeBucketsAndMaybeBreak(1);
+                }
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
index 9ac2d4aaccf..95bc83ad88f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
@@ -144,6 +144,9 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
             spare.bucketOrd = i;
             if (bucketCountThresholds.getShardMinDocCount() <= spare.docCount) {
                 spare = ordered.insertWithOverflow(spare);
+                if (spare == null) {
+                    consumeBucketsAndMaybeBreak(1);
+                }
             }
         }
 
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 38e582617a4..d143ebe3293 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -72,6 +72,7 @@ import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.SearchException;
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotException;
@@ -364,6 +365,14 @@ public class ExceptionSerializationTests extends ESTestCase {
         assertEquals(0, ex.getBytesWanted());
     }
 
+    public void testTooManyBucketsException() throws IOException {
+        MultiBucketConsumerService.TooManyBucketsException ex =
+            serialize(new MultiBucketConsumerService.TooManyBucketsException("Too many buckets", 100),
+                randomFrom(Version.V_7_0_0_alpha1));
+        assertEquals("Too many buckets", ex.getMessage());
+        assertEquals(100, ex.getMaxBuckets());
+    }
+
     public void testTimestampParsingException() throws IOException {
         TimestampParsingException ex = serialize(new TimestampParsingException("TIMESTAMP", null));
         assertEquals("failed to parse timestamp [TIMESTAMP]", ex.getMessage());
@@ -805,6 +814,7 @@ public class ExceptionSerializationTests extends ESTestCase {
         ids.put(146, org.elasticsearch.tasks.TaskCancelledException.class);
         ids.put(147, org.elasticsearch.env.ShardLockObtainFailedException.class);
         ids.put(148, org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException.class);
+        ids.put(149, MultiBucketConsumerService.TooManyBucketsException.class);
 
         Map, Integer> reverse = new HashMap<>();
         for (Map.Entry> entry : ids.entrySet()) {
diff --git a/core/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/core/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
index 15d24b85b49..c1f729a12ca 100644
--- a/core/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
@@ -56,7 +57,8 @@ public class DfsQueryPhaseTests extends ESTestCase {
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         SearchTransportService searchTransportService = new SearchTransportService(
             Settings.builder().put("search.remote.connect", false).build(), null, null) {
 
@@ -113,7 +115,8 @@ public class DfsQueryPhaseTests extends ESTestCase {
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         SearchTransportService searchTransportService = new SearchTransportService(
             Settings.builder().put("search.remote.connect", false).build(), null, null) {
 
@@ -169,7 +172,8 @@ public class DfsQueryPhaseTests extends ESTestCase {
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         SearchTransportService searchTransportService = new SearchTransportService(
             Settings.builder().put("search.remote.connect", false).build(), null, null) {
 
diff --git a/core/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/core/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index bd38a420f07..7f4fbc91157 100644
--- a/core/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.QueryFetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
@@ -44,7 +45,8 @@ import java.util.concurrent.atomic.AtomicReference;
 public class FetchSearchPhaseTests extends ESTestCase {
 
     public void testShortcutQueryAndFetchOptimization() throws IOException {
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
         InitialSearchPhase.ArraySearchPhaseResults results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 1);
@@ -85,7 +87,8 @@ public class FetchSearchPhaseTests extends ESTestCase {
 
     public void testFetchTwoDocument() throws IOException {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
         AtomicReference responseRef = new AtomicReference<>();
@@ -139,7 +142,8 @@ public class FetchSearchPhaseTests extends ESTestCase {
 
     public void testFailFetchOneDoc() throws IOException {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
         AtomicReference responseRef = new AtomicReference<>();
@@ -197,7 +201,8 @@ public class FetchSearchPhaseTests extends ESTestCase {
         int resultSetSize = randomIntBetween(0, 100);
         // we use at least 2 hits otherwise this is subject to single shard optimization and we trip an assert...
         int numHits = randomIntBetween(2, 100); // also numshards --> 1 hit per shard
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(numHits);
         InitialSearchPhase.ArraySearchPhaseResults results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), numHits);
@@ -253,7 +258,8 @@ public class FetchSearchPhaseTests extends ESTestCase {
 
     public void testExceptionFailsPhase() throws IOException {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
         AtomicReference responseRef = new AtomicReference<>();
@@ -306,7 +312,8 @@ public class FetchSearchPhaseTests extends ESTestCase {
 
     public void testCleanupIrrelevantContexts() throws IOException { // contexts that are not fetched should be cleaned up
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
         AtomicReference responseRef = new AtomicReference<>();
diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index 7501a7a90be..393c45fa572 100644
--- a/core/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -31,6 +31,7 @@ import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -66,7 +67,8 @@ public class SearchPhaseControllerTests extends ESTestCase {
 
     @Before
     public void setup() {
-        searchPhaseController = new SearchPhaseController(Settings.EMPTY, BigArrays.NON_RECYCLING_INSTANCE, null);
+        searchPhaseController = new SearchPhaseController(Settings.EMPTY,
+            (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
     }
 
     public void testSort() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
index 02b3632d2e8..2d9f462d862 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
@@ -41,6 +41,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.After;
+import org.junit.Before;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -90,6 +92,21 @@ public class EquivalenceIT extends ESIntegTestCase {
         }
     }
 
+    @Before
+    private void setupMaxBuckets() {
+        // disables the max bucket limit for this test
+        client().admin().cluster().prepareUpdateSettings()
+            .setTransientSettings(Collections.singletonMap("search.max_buckets", Integer.MAX_VALUE))
+            .get();
+    }
+
+    @After
+    private void cleanupMaxBuckets() {
+        client().admin().cluster().prepareUpdateSettings()
+            .setTransientSettings(Collections.singletonMap("search.max_buckets", null))
+            .get();
+    }
+
     // Make sure that unordered, reversed, disjoint and/or overlapping ranges are supported
     // Duel with filters
     public void testRandomRanges() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
index 73ebd0089b0..f359a3307bd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
@@ -1048,7 +1048,7 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
                 if (reduced) {
                     composite = searchAndReduce(indexSearcher, query, aggregationBuilder, FIELD_TYPES);
                 } else {
-                    composite = search(indexSearcher, query, aggregationBuilder, indexSettings, FIELD_TYPES);
+                    composite = search(indexSearcher, query, aggregationBuilder, FIELD_TYPES);
                 }
                 verify.accept(composite);
             }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
index 9d3374630a9..9b5bc7541f2 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
@@ -31,6 +31,7 @@ import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 
 import java.io.IOException;
@@ -39,6 +40,8 @@ import java.util.Collections;
 import java.util.List;
 import java.util.function.Consumer;
 
+import static org.elasticsearch.search.aggregations.MultiBucketConsumerService.TooManyBucketsException;
+
 public class DateHistogramAggregatorTests extends AggregatorTestCase {
 
     private static final String DATE_FIELD = "date";
@@ -335,28 +338,82 @@ public class DateHistogramAggregatorTests extends AggregatorTestCase {
         );
     }
 
+    public void testMaxBucket() throws IOException {
+        Query query = new MatchAllDocsQuery();
+        List timestamps = Arrays.asList(
+            "2010-01-01T00:00:00.000Z",
+            "2011-01-01T00:00:00.000Z",
+            "2017-01-01T00:00:00.000Z"
+        );
+
+        TooManyBucketsException exc = expectThrows(TooManyBucketsException.class, () -> testSearchCase(query, timestamps,
+            aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD),
+            histogram -> {}, 2));
+
+        exc = expectThrows(TooManyBucketsException.class, () -> testSearchAndReduceCase(query, timestamps,
+            aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD),
+            histogram -> {}, 2));
+
+        exc = expectThrows(TooManyBucketsException.class, () -> testSearchAndReduceCase(query, timestamps,
+            aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD).minDocCount(0L),
+            histogram -> {}, 100));
+
+        exc = expectThrows(TooManyBucketsException.class, () -> testSearchAndReduceCase(query, timestamps,
+            aggregation ->
+                aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5))
+                    .field(DATE_FIELD)
+                    .subAggregation(
+                        AggregationBuilders.dateHistogram("1")
+                            .dateHistogramInterval(DateHistogramInterval.seconds(5))
+                            .field(DATE_FIELD)
+                    ),
+            histogram -> {}, 5));
+    }
+
     private void testSearchCase(Query query, List dataset,
                                 Consumer configure,
                                 Consumer verify) throws IOException {
-        executeTestCase(false, query, dataset, configure, verify);
+        testSearchCase(query, dataset, configure, verify, 10000);
+    }
+
+    private void testSearchCase(Query query, List dataset,
+                                Consumer configure,
+                                Consumer verify,
+                                int maxBucket) throws IOException {
+        executeTestCase(false, query, dataset, configure, verify, maxBucket);
     }
 
     private void testSearchAndReduceCase(Query query, List dataset,
                                          Consumer configure,
                                          Consumer verify) throws IOException {
-        executeTestCase(true, query, dataset, configure, verify);
+        testSearchAndReduceCase(query, dataset, configure, verify, 1000);
+    }
+
+    private void testSearchAndReduceCase(Query query, List dataset,
+                                         Consumer configure,
+                                         Consumer verify,
+                                         int maxBucket) throws IOException {
+        executeTestCase(true, query, dataset, configure, verify, maxBucket);
     }
 
     private void testBothCases(Query query, List dataset,
                                Consumer configure,
                                Consumer verify) throws IOException {
-        testSearchCase(query, dataset, configure, verify);
-        testSearchAndReduceCase(query, dataset, configure, verify);
+        testBothCases(query, dataset, configure, verify, 10000);
+    }
+
+    private void testBothCases(Query query, List dataset,
+                               Consumer configure,
+                               Consumer verify,
+                               int maxBucket) throws IOException {
+        testSearchCase(query, dataset, configure, verify, maxBucket);
+        testSearchAndReduceCase(query, dataset, configure, verify, maxBucket);
     }
 
     private void executeTestCase(boolean reduced, Query query, List dataset,
                                  Consumer configure,
-                                 Consumer verify) throws IOException {
+                                 Consumer verify,
+                                 int maxBucket) throws IOException {
 
         try (Directory directory = newDirectory()) {
             try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
@@ -389,9 +446,9 @@ public class DateHistogramAggregatorTests extends AggregatorTestCase {
 
                 InternalDateHistogram histogram;
                 if (reduced) {
-                    histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType);
+                    histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, maxBucket, fieldType);
                 } else {
-                    histogram = search(indexSearcher, query, aggregationBuilder, fieldType);
+                    histogram = search(indexSearcher, query, aggregationBuilder, maxBucket, fieldType);
                 }
                 verify.accept(histogram);
             }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
index 47fccbc83c4..9dd355d8ca0 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
@@ -72,15 +72,14 @@ public class TermsAggregatorTests extends AggregatorTestCase {
 
     private boolean randomizeAggregatorImpl = true;
 
-    @Override
     protected  A createAggregator(AggregationBuilder aggregationBuilder,
-            IndexSearcher indexSearcher, IndexSettings indexSettings, MappedFieldType... fieldTypes) throws IOException {
+            IndexSearcher indexSearcher, MappedFieldType... fieldTypes) throws IOException {
         try {
             if (randomizeAggregatorImpl) {
                 TermsAggregatorFactory.COLLECT_SEGMENT_ORDS = randomBoolean();
                 TermsAggregatorFactory.REMAP_GLOBAL_ORDS = randomBoolean();
             }
-            return super.createAggregator(aggregationBuilder, indexSearcher, indexSettings, fieldTypes);
+            return super.createAggregator(aggregationBuilder, indexSearcher, fieldTypes);
         } finally {
             TermsAggregatorFactory.COLLECT_SEGMENT_ORDS = null;
             TermsAggregatorFactory.REMAP_GLOBAL_ORDS = null;
diff --git a/docs/reference/aggregations/bucket.asciidoc b/docs/reference/aggregations/bucket.asciidoc
index ed32aaed4c4..38036470457 100644
--- a/docs/reference/aggregations/bucket.asciidoc
+++ b/docs/reference/aggregations/bucket.asciidoc
@@ -13,6 +13,10 @@ aggregated for the buckets created by their "parent" bucket aggregation.
 There are different bucket aggregators, each with a different "bucketing" strategy. Some define a single bucket, some
 define fixed number of multiple buckets, and others dynamically create the buckets during the aggregation process.
 
+NOTE: The maximum number of buckets allowed in a single response is limited by a dynamic cluster
+setting named `search.max_buckets`. It defaults to 10,000, requests that try to return more than
+the limit will fail with an exception.
+
 include::bucket/adjacency-matrix-aggregation.asciidoc[]
 
 include::bucket/children-aggregation.asciidoc[]
diff --git a/docs/reference/migration/migrate_7_0/aggregations.asciidoc b/docs/reference/migration/migrate_7_0/aggregations.asciidoc
index 9f914972893..5241ba4ccc7 100644
--- a/docs/reference/migration/migrate_7_0/aggregations.asciidoc
+++ b/docs/reference/migration/migrate_7_0/aggregations.asciidoc
@@ -3,4 +3,10 @@
 
 ==== Deprecated `global_ordinals_hash` and `global_ordinals_low_cardinality` execution hints for terms aggregations have been removed
 
-These `execution_hint` are removed and should be replaced by `global_ordinals`.
\ No newline at end of file
+These `execution_hint` are removed and should be replaced by `global_ordinals`.
+
+==== `search.max_buckets` in the cluster setting
+
+The dynamic cluster setting named `search.max_buckets` now defaults
+to 10,000 (instead of unlimited in the previous version).
+Requests that try to return more than the limit will fail with an exception.
\ No newline at end of file
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml
new file mode 100644
index 00000000000..86c6c632d5e
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/240_max_buckets.yml
@@ -0,0 +1,110 @@
+---
+setup:
+  - do:
+        indices.create:
+          index: test
+          body:
+              mappings:
+                doc:
+                  properties:
+                    keyword:
+                      type: keyword
+                    date:
+                      type: date
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    1
+        body:  { "date": "2014-03-03T00:00:00", "keyword": "foo" }
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    2
+        body:  { "date": "2015-03-03T00:00:00", "keyword": "bar" }
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    3
+        body:  { "date": "2016-03-03T00:00:00", "keyword": "foobar" }
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    4
+        body:  { "date": "2017-03-03T00:00:00" }
+
+  - do:
+      indices.refresh:
+        index: [test]
+
+---
+ teardown:
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search.max_buckets: null
+
+---
+"Max bucket":
+  - skip:
+      version: " - 6.99.99"
+      reason:  search.max_buckets limit has been added in 7.0
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search.max_buckets: 3
+
+  - do:
+      catch: /.*Trying to create too many buckets.*/
+      search:
+        index: test
+        body:
+          aggregations:
+            test:
+              date_histogram:
+                field: date
+                interval: 1d
+
+  - do:
+      catch: /.*Trying to create too many buckets.*/
+      search:
+        index: test
+        body:
+          aggregations:
+            test:
+              terms:
+                field: keyword
+              aggs:
+                2:
+                  date_histogram:
+                    field: date
+                    interval: 1d
+
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            search.max_buckets: 100
+
+  - do:
+      catch: /.*Trying to create too many buckets.*/
+      search:
+        index: test
+        body:
+          aggregations:
+            test:
+              date_histogram:
+                field: date
+                interval: 1d
+                min_doc_count: 0
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index 7622aa6f217..8ae6c7843d2 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -59,17 +59,18 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.mock.orig.Mockito;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.subphase.FetchSourceSubPhase;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalAggregationTestCase;
 import org.junit.After;
 import org.mockito.Matchers;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -82,6 +83,7 @@ import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.elasticsearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS;
 
 /**
  * Base class for testing {@link Aggregator} implementations.
@@ -96,16 +98,20 @@ public abstract class AggregatorTestCase extends ESTestCase {
     protected AggregatorFactory createAggregatorFactory(AggregationBuilder aggregationBuilder,
                                                            IndexSearcher indexSearcher,
                                                            MappedFieldType... fieldTypes) throws IOException {
-        return createAggregatorFactory(aggregationBuilder, indexSearcher, createIndexSettings(), fieldTypes);
+        return createAggregatorFactory(aggregationBuilder, indexSearcher, createIndexSettings(),
+            new MultiBucketConsumer(DEFAULT_MAX_BUCKETS), fieldTypes);
     }
 
     /** Create a factory for the given aggregation builder. */
     protected AggregatorFactory createAggregatorFactory(AggregationBuilder aggregationBuilder,
                                                            IndexSearcher indexSearcher,
                                                            IndexSettings indexSettings,
+                                                           MultiBucketConsumer bucketConsumer,
                                                            MappedFieldType... fieldTypes) throws IOException {
         SearchContext searchContext = createSearchContext(indexSearcher, indexSettings);
         CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
+        when(searchContext.aggregations())
+            .thenReturn(new SearchContextAggregations(AggregatorFactories.EMPTY, bucketConsumer));
         when(searchContext.bigArrays()).thenReturn(new MockBigArrays(Settings.EMPTY, circuitBreakerService));
         // TODO: now just needed for top_hits, this will need to be revised for other agg unit tests:
         MapperService mapperService = mapperServiceMock();
@@ -116,12 +122,8 @@ public abstract class AggregatorTestCase extends ESTestCase {
         IndexFieldDataService ifds = new IndexFieldDataService(indexSettings,
                 new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() {
                 }), circuitBreakerService, mapperService);
-        when(searchContext.getForField(Mockito.any(MappedFieldType.class))).thenAnswer(new Answer() {
-            @Override
-            public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
-                return ifds.getForField((MappedFieldType) invocationOnMock.getArguments()[0]);
-            }
-        });
+        when(searchContext.getForField(Mockito.any(MappedFieldType.class)))
+            .thenAnswer(invocationOnMock -> ifds.getForField((MappedFieldType) invocationOnMock.getArguments()[0]));
 
         SearchLookup searchLookup = new SearchLookup(mapperService, ifds::getForField, new String[]{TYPE_NAME});
         when(searchContext.lookup()).thenReturn(searchLookup);
@@ -139,15 +141,32 @@ public abstract class AggregatorTestCase extends ESTestCase {
     protected  A createAggregator(AggregationBuilder aggregationBuilder,
                                                         IndexSearcher indexSearcher,
                                                         MappedFieldType... fieldTypes) throws IOException {
-        return createAggregator(aggregationBuilder, indexSearcher, createIndexSettings(), fieldTypes);
+        return createAggregator(aggregationBuilder, indexSearcher, createIndexSettings(),
+            new MultiBucketConsumer(DEFAULT_MAX_BUCKETS), fieldTypes);
     }
 
     protected  A createAggregator(AggregationBuilder aggregationBuilder,
                                                         IndexSearcher indexSearcher,
                                                         IndexSettings indexSettings,
                                                         MappedFieldType... fieldTypes) throws IOException {
+        return createAggregator(aggregationBuilder, indexSearcher, indexSettings,
+            new MultiBucketConsumer(DEFAULT_MAX_BUCKETS), fieldTypes);
+    }
+
+    protected  A createAggregator(AggregationBuilder aggregationBuilder,
+                                                        IndexSearcher indexSearcher,
+                                                        MultiBucketConsumer bucketConsumer,
+                                                        MappedFieldType... fieldTypes) throws IOException {
+        return createAggregator(aggregationBuilder, indexSearcher, createIndexSettings(), bucketConsumer, fieldTypes);
+    }
+
+    protected  A createAggregator(AggregationBuilder aggregationBuilder,
+                                                        IndexSearcher indexSearcher,
+                                                        IndexSettings indexSettings,
+                                                        MultiBucketConsumer bucketConsumer,
+                                                        MappedFieldType... fieldTypes) throws IOException {
         @SuppressWarnings("unchecked")
-        A aggregator = (A) createAggregatorFactory(aggregationBuilder, indexSearcher, indexSettings, fieldTypes)
+        A aggregator = (A) createAggregatorFactory(aggregationBuilder, indexSearcher, indexSettings, bucketConsumer, fieldTypes)
             .create(null, true);
         return aggregator;
     }
@@ -233,24 +252,33 @@ public abstract class AggregatorTestCase extends ESTestCase {
                                                                              Query query,
                                                                              AggregationBuilder builder,
                                                                              MappedFieldType... fieldTypes) throws IOException {
-        return search(searcher, query, builder, createIndexSettings(), fieldTypes);
+        return search(searcher, query, builder, DEFAULT_MAX_BUCKETS, fieldTypes);
     }
 
     protected  A search(IndexSearcher searcher,
                                                                              Query query,
                                                                              AggregationBuilder builder,
-                                                                             IndexSettings indexSettings,
+                                                                             int maxBucket,
                                                                              MappedFieldType... fieldTypes) throws IOException {
-        C a = createAggregator(builder, searcher, fieldTypes);
+        MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket);
+        C a = createAggregator(builder, searcher, bucketConsumer, fieldTypes);
         a.preCollection();
         searcher.search(query, a);
         a.postCollection();
         @SuppressWarnings("unchecked")
         A internalAgg = (A) a.buildAggregation(0L);
+        InternalAggregationTestCase.assertMultiBucketConsumer(internalAgg, bucketConsumer);
         return internalAgg;
 
     }
 
+    protected  A searchAndReduce(IndexSearcher searcher,
+                                                                                      Query query,
+                                                                                      AggregationBuilder builder,
+                                                                                      MappedFieldType... fieldTypes) throws IOException {
+        return searchAndReduce(searcher, query, builder, DEFAULT_MAX_BUCKETS, fieldTypes);
+    }
+
     /**
      * Divides the provided {@link IndexSearcher} in sub-searcher, one for each segment,
      * builds an aggregator for each sub-searcher filtered by the provided {@link Query} and
@@ -259,6 +287,7 @@ public abstract class AggregatorTestCase extends ESTestCase {
     protected  A searchAndReduce(IndexSearcher searcher,
                                                                                       Query query,
                                                                                       AggregationBuilder builder,
+                                                                                      int maxBucket,
                                                                                       MappedFieldType... fieldTypes) throws IOException {
         final IndexReaderContext ctx = searcher.getTopReaderContext();
 
@@ -279,14 +308,18 @@ public abstract class AggregatorTestCase extends ESTestCase {
         List aggs = new ArrayList<> ();
         Query rewritten = searcher.rewrite(query);
         Weight weight = searcher.createWeight(rewritten, true, 1f);
-        C root = createAggregator(builder, searcher, fieldTypes);
+        MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket);
+        C root = createAggregator(builder, searcher, bucketConsumer, fieldTypes);
 
         for (ShardSearcher subSearcher : subSearchers) {
-            C a = createAggregator(builder, subSearcher, fieldTypes);
+            MultiBucketConsumer shardBucketConsumer = new MultiBucketConsumer(maxBucket);
+            C a = createAggregator(builder, subSearcher, shardBucketConsumer, fieldTypes);
             a.preCollection();
             subSearcher.search(weight, a);
             a.postCollection();
-            aggs.add(a.buildAggregation(0L));
+            InternalAggregation agg = a.buildAggregation(0L);
+            aggs.add(agg);
+            InternalAggregationTestCase.assertMultiBucketConsumer(agg, shardBucketConsumer);
         }
         if (aggs.isEmpty()) {
             return null;
@@ -297,15 +330,23 @@ public abstract class AggregatorTestCase extends ESTestCase {
                 Collections.shuffle(aggs, random());
                 int r = randomIntBetween(1, toReduceSize);
                 List toReduce = aggs.subList(0, r);
-                A reduced = (A) aggs.get(0).doReduce(toReduce,
-                    new InternalAggregation.ReduceContext(root.context().bigArrays(), null, false));
+                MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer(maxBucket);
+                InternalAggregation.ReduceContext context =
+                    new InternalAggregation.ReduceContext(root.context().bigArrays(), null,
+                        reduceBucketConsumer, false);
+                A reduced = (A) aggs.get(0).doReduce(toReduce, context);
+                InternalAggregationTestCase.assertMultiBucketConsumer(reduced, reduceBucketConsumer);
                 aggs = new ArrayList<>(aggs.subList(r, toReduceSize));
                 aggs.add(reduced);
             }
             // now do the final reduce
+            MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer(maxBucket);
+            InternalAggregation.ReduceContext context =
+                new InternalAggregation.ReduceContext(root.context().bigArrays(), null, reduceBucketConsumer, true);
+
             @SuppressWarnings("unchecked")
-            A internalAgg = (A) aggs.get(0).doReduce(aggs, new InternalAggregation.ReduceContext(root.context().bigArrays(), null,
-                true));
+            A internalAgg = (A) aggs.get(0).doReduce(aggs, context);
+            InternalAggregationTestCase.assertMultiBucketConsumer(internalAgg, reduceBucketConsumer);
             return internalAgg;
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
index a69695fa183..9de83aee630 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
@@ -38,7 +38,9 @@ import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer;
 import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
 import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.adjacency.ParsedAdjacencyMatrix;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
@@ -140,10 +142,13 @@ import java.util.stream.Collectors;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.search.aggregations.InternalMultiBucketAggregation.countInnerBucket;
 import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+import static org.hamcrest.Matchers.equalTo;
 
 public abstract class InternalAggregationTestCase extends AbstractWireSerializingTestCase {
+    public static final int DEFAULT_MAX_BUCKETS = 100000;
 
     private final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(
             new SearchModule(Settings.EMPTY, false, emptyList()).getNamedWriteables());
@@ -237,17 +242,21 @@ public abstract class InternalAggregationTestCase
             Collections.shuffle(toReduce, random());
             int r = randomIntBetween(1, toReduceSize);
             List internalAggregations = toReduce.subList(0, r);
+            MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(DEFAULT_MAX_BUCKETS);
             InternalAggregation.ReduceContext context =
-                new InternalAggregation.ReduceContext(bigArrays, mockScriptService, false);
+                new InternalAggregation.ReduceContext(bigArrays, mockScriptService, bucketConsumer,false);
             @SuppressWarnings("unchecked")
             T reduced = (T) inputs.get(0).reduce(internalAggregations, context);
+            assertMultiBucketConsumer(reduced, bucketConsumer);
             toReduce = new ArrayList<>(toReduce.subList(r, toReduceSize));
             toReduce.add(reduced);
         }
+        MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(DEFAULT_MAX_BUCKETS);
         InternalAggregation.ReduceContext context =
-            new InternalAggregation.ReduceContext(bigArrays, mockScriptService, true);
+            new InternalAggregation.ReduceContext(bigArrays, mockScriptService, bucketConsumer, true);
         @SuppressWarnings("unchecked")
         T reduced = (T) inputs.get(0).reduce(toReduce, context);
+        assertMultiBucketConsumer(reduced, bucketConsumer);
         assertReduced(reduced, inputs);
     }
 
@@ -392,4 +401,8 @@ public abstract class InternalAggregationTestCase
         formats.add(() -> new DocValueFormat.Decimal(randomFrom("###.##", "###,###.##")));
         return randomFrom(formats).get();
     }
+
+    public static void assertMultiBucketConsumer(Aggregation agg, MultiBucketConsumer bucketConsumer) {
+        assertThat(bucketConsumer.getCount(), equalTo(countInnerBucket(agg)));
+    }
 }

From e0e698bc26244c291917a1e771185f823992e80f Mon Sep 17 00:00:00 2001
From: Boaz Leskes 
Date: Wed, 6 Dec 2017 14:46:39 +0100
Subject: [PATCH 197/297] testCorruptTranslogTruncation: add logging

---
 .../org/elasticsearch/index/translog/TruncateTranslogIT.java    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
index d98359cdd06..90fe9e8404b 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
@@ -57,6 +57,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.engine.MockEngineSupport;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 
 import java.io.IOException;
@@ -84,6 +85,7 @@ public class TruncateTranslogIT extends ESIntegTestCase {
         return Arrays.asList(MockTransportService.TestPlugin.class, MockEngineFactoryPlugin.class);
     }
 
+    @TestLogging("org.elasticsearch.index.engine:TRACE,org.elasticsearch.index.translog:TRACE")
     public void testCorruptTranslogTruncation() throws Exception {
         internalCluster().startNodes(2, Settings.EMPTY);
 

From 2aa62daed436b827c4315ba4e038b835af203b97 Mon Sep 17 00:00:00 2001
From: Tim Brooks 
Date: Wed, 6 Dec 2017 11:02:25 -0700
Subject: [PATCH 198/297] Introduce resizable inbound byte buffer (#27551)

This is related to #27563. In order to interface with java nio, we must
have buffers that are compatible with ByteBuffer. This commit introduces
a basic ByteBufferReference to easily allow transferring bytes off the
wire to usage in the application.

Additionally it introduces an InboundChannelBuffer. This is a buffer
that can internally expand as more space is needed. It is designed to
be integrated with a page recycler so that it can internally reuse pages.
The final piece is moving all of the index work for writing bytes to a
channel into the WriteOperation.
---
 .../common/bytes/ByteBufferReference.java     |  89 ++++++++
 .../bytes/ByteBufferReferenceTests.java       |  44 ++++
 .../transport/nio/InboundChannelBuffer.java   | 204 ++++++++++++++++++
 .../transport/nio/NetworkBytesReference.java  | 157 --------------
 .../transport/nio/WriteOperation.java         |  60 ++++--
 .../nio/channel/NioSocketChannel.java         |  29 +--
 .../nio/channel/TcpFrameDecoder.java          |   6 +-
 .../transport/nio/channel/TcpReadContext.java |  58 ++---
 .../nio/ByteBufferReferenceTests.java         | 155 -------------
 .../nio/InboundChannelBufferTests.java        | 152 +++++++++++++
 .../nio/SocketEventHandlerTests.java          |   6 +-
 .../transport/nio/SocketSelectorTests.java    |   6 +-
 .../transport/nio/WriteOperationTests.java    |  14 +-
 .../nio/channel/TcpFrameDecoderTests.java     |  18 +-
 .../nio/channel/TcpReadContextTests.java      |  32 +--
 .../nio/channel/TcpWriteContextTests.java     |   4 +-
 16 files changed, 601 insertions(+), 433 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/common/bytes/ByteBufferReference.java
 create mode 100644 core/src/test/java/org/elasticsearch/common/bytes/ByteBufferReferenceTests.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/transport/nio/InboundChannelBuffer.java
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/transport/nio/NetworkBytesReference.java
 delete mode 100644 test/framework/src/test/java/org/elasticsearch/transport/nio/ByteBufferReferenceTests.java
 create mode 100644 test/framework/src/test/java/org/elasticsearch/transport/nio/InboundChannelBufferTests.java

diff --git a/core/src/main/java/org/elasticsearch/common/bytes/ByteBufferReference.java b/core/src/main/java/org/elasticsearch/common/bytes/ByteBufferReference.java
new file mode 100644
index 00000000000..fbdcdfd6885
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/bytes/ByteBufferReference.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.bytes;
+
+import org.apache.lucene.util.BytesRef;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This is a {@link BytesReference} backed by a {@link ByteBuffer}. The byte buffer can either be a heap or
+ * direct byte buffer. The reference is composed of the space between the {@link ByteBuffer#position} and
+ * {@link ByteBuffer#limit} at construction time. If the position or limit of the underlying byte buffer is
+ * changed, those changes will not be reflected in this reference. However, modifying the limit or position
+ * of the underlying byte buffer is not recommended as those can be used during {@link ByteBuffer#get()}
+ * bounds checks. Use {@link ByteBuffer#duplicate()} at creation time if you plan on modifying the markers of
+ * the underlying byte buffer. Any changes to the underlying data in the byte buffer will be reflected.
+ */
+public class ByteBufferReference extends BytesReference {
+
+    private final ByteBuffer buffer;
+    private final int offset;
+    private final int length;
+
+    public ByteBufferReference(ByteBuffer buffer) {
+        this.buffer = buffer;
+        this.offset = buffer.position();
+        this.length = buffer.remaining();
+    }
+
+    @Override
+    public byte get(int index) {
+        return buffer.get(index + offset);
+    }
+
+    @Override
+    public int length() {
+        return length;
+    }
+
+    @Override
+    public BytesReference slice(int from, int length) {
+        if (from < 0 || (from + length) > this.length) {
+            throw new IndexOutOfBoundsException("can't slice a buffer with length [" + this.length + "], with slice parameters from ["
+                + from + "], length [" + length + "]");
+        }
+        ByteBuffer newByteBuffer = buffer.duplicate();
+        newByteBuffer.position(offset + from);
+        newByteBuffer.limit(offset + from + length);
+        return new ByteBufferReference(newByteBuffer);
+    }
+
+    /**
+     * This will return a bytes ref composed of the bytes. If this is a direct byte buffer, the bytes will
+     * have to be copied.
+     *
+     * @return the bytes ref
+     */
+    @Override
+    public BytesRef toBytesRef() {
+        if (buffer.hasArray()) {
+            return new BytesRef(buffer.array(), buffer.arrayOffset() + offset, length);
+        }
+        final byte[] copy = new byte[length];
+        buffer.get(copy, offset, length);
+        return new BytesRef(copy);
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        return buffer.capacity();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/ByteBufferReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/ByteBufferReferenceTests.java
new file mode 100644
index 00000000000..9560fd40038
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/bytes/ByteBufferReferenceTests.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.bytes;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class ByteBufferReferenceTests extends AbstractBytesReferenceTestCase {
+
+    private void initializeBytes(byte[] bytes) {
+        for (int i = 0 ; i < bytes.length; ++i) {
+            bytes[i] = (byte) i;
+        }
+    }
+
+    @Override
+    protected BytesReference newBytesReference(int length) throws IOException {
+        return newBytesReferenceWithOffsetOfZero(length);
+    }
+
+    @Override
+    protected BytesReference newBytesReferenceWithOffsetOfZero(int length) throws IOException {
+        byte[] bytes = new byte[length];
+        initializeBytes(bytes);
+        return new ByteBufferReference(ByteBuffer.wrap(bytes));
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/InboundChannelBuffer.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/InboundChannelBuffer.java
new file mode 100644
index 00000000000..46cec52bb6c
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/InboundChannelBuffer.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport.nio;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Iterator;
+import java.util.function.Supplier;
+
+/**
+ * This is a channel byte buffer composed internally of 16kb pages. When an entire message has been read
+ * and consumed, the {@link #release(long)} method releases the bytes from the head of the buffer and closes
+ * the pages internally. If more space is needed at the end of the buffer {@link #ensureCapacity(long)} can
+ * be called and the buffer will expand using the supplier provided.
+ */
+public final class InboundChannelBuffer {
+
+    private static final int PAGE_SIZE = 1 << 14;
+    private static final int PAGE_MASK = PAGE_SIZE - 1;
+    private static final int PAGE_SHIFT = Integer.numberOfTrailingZeros(PAGE_SIZE);
+    private static final ByteBuffer[] EMPTY_BYTE_BUFFER_ARRAY = new ByteBuffer[0];
+
+
+    private final ArrayDeque pages;
+    private final Supplier pageSupplier;
+
+    private long capacity = 0;
+    private long internalIndex = 0;
+    // The offset is an int as it is the offset of where the bytes begin in the first buffer
+    private int offset = 0;
+
+    public InboundChannelBuffer() {
+        this(() -> ByteBuffer.wrap(new byte[PAGE_SIZE]));
+    }
+
+    private InboundChannelBuffer(Supplier pageSupplier) {
+        this.pageSupplier = pageSupplier;
+        this.pages = new ArrayDeque<>();
+        this.capacity = PAGE_SIZE * pages.size();
+        ensureCapacity(PAGE_SIZE);
+    }
+
+    public void ensureCapacity(long requiredCapacity) {
+        if (capacity < requiredCapacity) {
+            int numPages = numPages(requiredCapacity + offset);
+            int pagesToAdd = numPages - pages.size();
+            for (int i = 0; i < pagesToAdd; i++) {
+                pages.addLast(pageSupplier.get());
+            }
+            capacity += pagesToAdd * PAGE_SIZE;
+        }
+    }
+
+    /**
+     * This method will release bytes from the head of this buffer. If you release bytes past the current
+     * index the index is truncated to zero.
+     *
+     * @param bytesToRelease number of bytes to drop
+     */
+    public void release(long bytesToRelease) {
+        if (bytesToRelease > capacity) {
+            throw new IllegalArgumentException("Releasing more bytes [" + bytesToRelease + "] than buffer capacity [" + capacity + "].");
+        }
+
+        int pagesToRelease = pageIndex(offset + bytesToRelease);
+        for (int i = 0; i < pagesToRelease; i++) {
+            pages.removeFirst();
+        }
+        capacity -= bytesToRelease;
+        internalIndex = Math.max(internalIndex - bytesToRelease, 0);
+        offset = indexInPage(bytesToRelease + offset);
+    }
+
+    /**
+     * This method will return an array of {@link ByteBuffer} representing the bytes from the beginning of
+     * this buffer up through the index argument that was passed. The buffers will be duplicates of the
+     * internal buffers, so any modifications to the markers {@link ByteBuffer#position()},
+     * {@link ByteBuffer#limit()}, etc will not modify the this class.
+     *
+     * @param to the index to slice up to
+     * @return the byte buffers
+     */
+    public ByteBuffer[] sliceBuffersTo(long to) {
+        if (to > capacity) {
+            throw new IndexOutOfBoundsException("can't slice a channel buffer with capacity [" + capacity +
+                "], with slice parameters to [" + to + "]");
+        } else if (to == 0) {
+            return EMPTY_BYTE_BUFFER_ARRAY;
+        }
+        long indexWithOffset = to + offset;
+        int pageCount = pageIndex(indexWithOffset);
+        int finalLimit = indexInPage(indexWithOffset);
+        if (finalLimit != 0) {
+            pageCount += 1;
+        }
+
+        ByteBuffer[] buffers = new ByteBuffer[pageCount];
+        Iterator pageIterator = pages.iterator();
+        ByteBuffer firstBuffer = pageIterator.next().duplicate();
+        firstBuffer.position(firstBuffer.position() + offset);
+        buffers[0] = firstBuffer;
+        for (int i = 1; i < buffers.length; i++) {
+            buffers[i] = pageIterator.next().duplicate();
+        }
+        if (finalLimit != 0) {
+            buffers[buffers.length - 1].limit(finalLimit);
+        }
+
+        return buffers;
+    }
+
+    /**
+     * This method will return an array of {@link ByteBuffer} representing the bytes from the index passed
+     * through the end of this buffer. The buffers will be duplicates of the internal buffers, so any
+     * modifications to the markers {@link ByteBuffer#position()}, {@link ByteBuffer#limit()}, etc will not
+     * modify the this class.
+     *
+     * @param from the index to slice from
+     * @return the byte buffers
+     */
+    public ByteBuffer[] sliceBuffersFrom(long from) {
+        if (from > capacity) {
+            throw new IndexOutOfBoundsException("can't slice a channel buffer with capacity [" + capacity +
+                "], with slice parameters from [" + from + "]");
+        } else if (from == capacity) {
+            return EMPTY_BYTE_BUFFER_ARRAY;
+        }
+        long indexWithOffset = from + offset;
+
+        int pageIndex = pageIndex(indexWithOffset);
+        int indexInPage = indexInPage(indexWithOffset);
+
+        ByteBuffer[] buffers = new ByteBuffer[pages.size() - pageIndex];
+        Iterator pageIterator = pages.descendingIterator();
+        for (int i = buffers.length - 1; i > 0; --i) {
+            buffers[i] = pageIterator.next().duplicate();
+        }
+        ByteBuffer firstPostIndexBuffer = pageIterator.next().duplicate();
+        firstPostIndexBuffer.position(firstPostIndexBuffer.position() + indexInPage);
+        buffers[0] = firstPostIndexBuffer;
+
+        return buffers;
+    }
+
+    public void incrementIndex(long delta) {
+        if (delta < 0) {
+            throw new IllegalArgumentException("Cannot increment an index with a negative delta [" + delta + "]");
+        }
+
+        long newIndex = delta + internalIndex;
+        if (newIndex > capacity) {
+            throw new IllegalArgumentException("Cannot increment an index [" + internalIndex + "] with a delta [" + delta +
+                "] that will result in a new index [" + newIndex + "] that is greater than the capacity [" + capacity + "].");
+        }
+        internalIndex = newIndex;
+    }
+
+    public long getIndex() {
+        return internalIndex;
+    }
+
+    public long getCapacity() {
+        return capacity;
+    }
+
+    public long getRemaining() {
+        long remaining = capacity - internalIndex;
+        assert remaining >= 0 : "The remaining [" + remaining + "] number of bytes should not be less than zero.";
+        return remaining;
+    }
+
+    private int numPages(long capacity) {
+        final long numPages = (capacity + PAGE_MASK) >>> PAGE_SHIFT;
+        if (numPages > Integer.MAX_VALUE) {
+            throw new IllegalArgumentException("pageSize=" + (PAGE_MASK + 1) + " is too small for such as capacity: " + capacity);
+        }
+        return (int) numPages;
+    }
+
+    private int pageIndex(long index) {
+        return (int) (index >>> PAGE_SHIFT);
+    }
+
+    private int indexInPage(long index) {
+        return (int) (index & PAGE_MASK);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/NetworkBytesReference.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/NetworkBytesReference.java
deleted file mode 100644
index cbccd7333d6..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/NetworkBytesReference.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.nio;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-
-public class NetworkBytesReference extends BytesReference {
-
-    private final BytesArray bytesArray;
-    private final ByteBuffer writeBuffer;
-    private final ByteBuffer readBuffer;
-
-    private int writeIndex;
-    private int readIndex;
-
-    public NetworkBytesReference(BytesArray bytesArray, int writeIndex, int readIndex) {
-        this.bytesArray = bytesArray;
-        this.writeIndex = writeIndex;
-        this.readIndex = readIndex;
-        this.writeBuffer = ByteBuffer.wrap(bytesArray.array());
-        this.readBuffer = ByteBuffer.wrap(bytesArray.array());
-    }
-
-    public static NetworkBytesReference wrap(BytesArray bytesArray) {
-        return wrap(bytesArray, 0, 0);
-    }
-
-    public static NetworkBytesReference wrap(BytesArray bytesArray, int writeIndex, int readIndex) {
-        if (readIndex > writeIndex) {
-            throw new IndexOutOfBoundsException("Read index [" + readIndex + "] was greater than write index [" + writeIndex + "]");
-        }
-        return new NetworkBytesReference(bytesArray, writeIndex, readIndex);
-    }
-
-    @Override
-    public byte get(int index) {
-        return bytesArray.get(index);
-    }
-
-    @Override
-    public int length() {
-        return bytesArray.length();
-    }
-
-    @Override
-    public NetworkBytesReference slice(int from, int length) {
-        BytesReference ref = bytesArray.slice(from, length);
-        BytesArray newBytesArray;
-        if (ref instanceof BytesArray) {
-            newBytesArray = (BytesArray) ref;
-        } else {
-            newBytesArray = new BytesArray(ref.toBytesRef());
-        }
-
-        int newReadIndex = Math.min(Math.max(readIndex - from, 0), length);
-        int newWriteIndex = Math.min(Math.max(writeIndex - from, 0), length);
-
-        return wrap(newBytesArray, newWriteIndex, newReadIndex);
-    }
-
-    @Override
-    public BytesRef toBytesRef() {
-        return bytesArray.toBytesRef();
-    }
-
-    @Override
-    public long ramBytesUsed() {
-        return bytesArray.ramBytesUsed();
-    }
-
-    public int getWriteIndex() {
-        return writeIndex;
-    }
-
-    public void incrementWrite(int delta) {
-        int newWriteIndex = writeIndex + delta;
-        if (newWriteIndex > bytesArray.length()) {
-            throw new IndexOutOfBoundsException("New write index [" + newWriteIndex + "] would be greater than length" +
-                " [" + bytesArray.length() + "]");
-        }
-
-        writeIndex = newWriteIndex;
-    }
-
-    public int getWriteRemaining() {
-        return bytesArray.length() - writeIndex;
-    }
-
-    public boolean hasWriteRemaining() {
-        return getWriteRemaining() > 0;
-    }
-
-    public int getReadIndex() {
-        return readIndex;
-    }
-
-    public void incrementRead(int delta) {
-        int newReadIndex = readIndex + delta;
-        if (newReadIndex > writeIndex) {
-            throw new IndexOutOfBoundsException("New read index [" + newReadIndex + "] would be greater than write" +
-                " index [" + writeIndex + "]");
-        }
-        readIndex = newReadIndex;
-    }
-
-    public int getReadRemaining() {
-        return writeIndex - readIndex;
-    }
-
-    public boolean hasReadRemaining() {
-        return getReadRemaining() > 0;
-    }
-
-    public ByteBuffer getWriteByteBuffer() {
-        writeBuffer.position(bytesArray.offset() + writeIndex);
-        writeBuffer.limit(bytesArray.offset() + bytesArray.length());
-        return writeBuffer;
-    }
-
-    public ByteBuffer getReadByteBuffer() {
-        readBuffer.position(bytesArray.offset() + readIndex);
-        readBuffer.limit(bytesArray.offset() + writeIndex);
-        return readBuffer;
-    }
-
-    public static void vectorizedIncrementReadIndexes(Iterable references, int delta) {
-        Iterator refs = references.iterator();
-        while (delta != 0) {
-            NetworkBytesReference ref = refs.next();
-            int amountToInc = Math.min(ref.getReadRemaining(), delta);
-            ref.incrementRead(amountToInc);
-            delta -= amountToInc;
-        }
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/WriteOperation.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/WriteOperation.java
index f91acc5bbea..0abb6a67650 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/WriteOperation.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/WriteOperation.java
@@ -27,22 +27,35 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.transport.nio.channel.NioSocketChannel;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Arrays;
 
 public class WriteOperation {
 
     private final NioSocketChannel channel;
     private final ActionListener listener;
-    private final NetworkBytesReference[] references;
+    private final ByteBuffer[] buffers;
+    private final int[] offsets;
+    private final int length;
+    private int internalIndex;
 
     public WriteOperation(NioSocketChannel channel, BytesReference bytesReference, ActionListener listener) {
         this.channel = channel;
         this.listener = listener;
-        this.references = toArray(bytesReference);
+        this.buffers = toByteBuffers(bytesReference);
+        this.offsets = new int[buffers.length];
+        int offset = 0;
+        for (int i = 0; i < buffers.length; i++) {
+            ByteBuffer buffer = buffers[i];
+            offsets[i] = offset;
+            offset += buffer.remaining();
+        }
+        length = offset;
     }
 
-    public NetworkBytesReference[] getByteReferences() {
-        return references;
+    public ByteBuffer[] getByteBuffers() {
+        return buffers;
     }
 
     public ActionListener getListener() {
@@ -54,23 +67,46 @@ public class WriteOperation {
     }
 
     public boolean isFullyFlushed() {
-        return references[references.length - 1].hasReadRemaining() == false;
+        return internalIndex == length;
     }
 
     public int flush() throws IOException {
-        return channel.write(references);
+        int written = channel.write(getBuffersToWrite());
+        internalIndex += written;
+        return written;
     }
 
-    private static NetworkBytesReference[] toArray(BytesReference reference) {
-        BytesRefIterator byteRefIterator = reference.iterator();
+    private ByteBuffer[] getBuffersToWrite() {
+        int offsetIndex = getOffsetIndex(internalIndex);
+
+        ByteBuffer[] postIndexBuffers = new ByteBuffer[buffers.length - offsetIndex];
+
+        ByteBuffer firstBuffer = buffers[0].duplicate();
+        firstBuffer.position(internalIndex - offsets[offsetIndex]);
+        postIndexBuffers[offsetIndex] = firstBuffer;
+        int j = 1;
+        for (int i = (offsetIndex + 1); i < buffers.length; ++i) {
+            postIndexBuffers[j++] = buffers[i].duplicate();
+        }
+
+        return postIndexBuffers;
+    }
+
+    private int getOffsetIndex(int offset) {
+        final int i = Arrays.binarySearch(offsets, offset);
+        return i < 0 ? (-(i + 1)) - 1 : i;
+    }
+
+    private static ByteBuffer[] toByteBuffers(BytesReference bytesReference) {
+        BytesRefIterator byteRefIterator = bytesReference.iterator();
         BytesRef r;
         try {
-            // Most network messages are composed of three buffers
-            ArrayList references = new ArrayList<>(3);
+            // Most network messages are composed of three buffers.
+            ArrayList buffers = new ArrayList<>(3);
             while ((r = byteRefIterator.next()) != null) {
-                references.add(NetworkBytesReference.wrap(new BytesArray(r), r.length, 0));
+                buffers.add(ByteBuffer.wrap(r.bytes, r.offset, r.length));
             }
-            return references.toArray(new NetworkBytesReference[references.size()]);
+            return buffers.toArray(new ByteBuffer[buffers.size()]);
 
         } catch (IOException e) {
             // this is really an error since we don't do IO in our bytesreferences
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/NioSocketChannel.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/NioSocketChannel.java
index b56731aee10..0f6c6715088 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/NioSocketChannel.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/NioSocketChannel.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.transport.nio.channel;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.transport.nio.NetworkBytesReference;
+import org.elasticsearch.transport.nio.InboundChannelBuffer;
 import org.elasticsearch.transport.nio.SocketSelector;
 
 import java.io.IOException;
@@ -28,7 +28,6 @@ import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.nio.channels.ClosedChannelException;
 import java.nio.channels.SocketChannel;
-import java.util.Arrays;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
@@ -66,34 +65,22 @@ public class NioSocketChannel extends AbstractNioChannel {
         return socketSelector;
     }
 
-    public int write(NetworkBytesReference[] references) throws IOException {
-        int written;
-        if (references.length == 1) {
-            written = socketChannel.write(references[0].getReadByteBuffer());
+    public int write(ByteBuffer[] buffers) throws IOException {
+        if (buffers.length == 1) {
+            return socketChannel.write(buffers[0]);
         } else {
-            ByteBuffer[] buffers = new ByteBuffer[references.length];
-            for (int i = 0; i < references.length; ++i) {
-                buffers[i] = references[i].getReadByteBuffer();
-            }
-            written = (int) socketChannel.write(buffers);
+            return (int) socketChannel.write(buffers);
         }
-        if (written <= 0) {
-            return written;
-        }
-
-        NetworkBytesReference.vectorizedIncrementReadIndexes(Arrays.asList(references), written);
-
-        return written;
     }
 
-    public int read(NetworkBytesReference reference) throws IOException {
-        int bytesRead = socketChannel.read(reference.getWriteByteBuffer());
+    public int read(InboundChannelBuffer buffer) throws IOException {
+        int bytesRead = (int) socketChannel.read(buffer.sliceBuffersFrom(buffer.getIndex()));
 
         if (bytesRead == -1) {
             return bytesRead;
         }
 
-        reference.incrementWrite(bytesRead);
+        buffer.incrementIndex(bytesRead);
         return bytesRead;
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpFrameDecoder.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpFrameDecoder.java
index 356af44c5ba..b2ba70fb236 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpFrameDecoder.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpFrameDecoder.java
@@ -36,11 +36,11 @@ public class TcpFrameDecoder {
 
     private int expectedMessageLength = -1;
 
-    public BytesReference decode(BytesReference bytesReference, int currentBufferSize) throws IOException {
-        if (currentBufferSize >= 6) {
+    public BytesReference decode(BytesReference bytesReference) throws IOException {
+        if (bytesReference.length() >= 6) {
             int messageLength = readHeaderBuffer(bytesReference);
             int totalLength = messageLength + HEADER_SIZE;
-            if (totalLength > currentBufferSize) {
+            if (totalLength > bytesReference.length()) {
                 expectedMessageLength = totalLength;
                 return null;
             } else if (totalLength == bytesReference.length()) {
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpReadContext.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpReadContext.java
index 8eeb32a976c..ae9fe0fdc93 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpReadContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpReadContext.java
@@ -19,25 +19,21 @@
 
 package org.elasticsearch.transport.nio.channel;
 
-import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.ByteBufferReference;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.CompositeBytesReference;
-import org.elasticsearch.transport.nio.NetworkBytesReference;
+import org.elasticsearch.transport.nio.InboundChannelBuffer;
 import org.elasticsearch.transport.nio.TcpReadHandler;
 
 import java.io.IOException;
-import java.util.Iterator;
-import java.util.LinkedList;
+import java.nio.ByteBuffer;
 
 public class TcpReadContext implements ReadContext {
 
-    private static final int DEFAULT_READ_LENGTH = 1 << 14;
-
     private final TcpReadHandler handler;
     private final TcpNioSocketChannel channel;
     private final TcpFrameDecoder frameDecoder;
-    private final LinkedList references = new LinkedList<>();
-    private int rawBytesCount = 0;
+    private final InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
 
     public TcpReadContext(NioSocketChannel channel, TcpReadHandler handler) {
         this((TcpNioSocketChannel) channel, handler, new TcpFrameDecoder());
@@ -47,33 +43,27 @@ public class TcpReadContext implements ReadContext {
         this.handler = handler;
         this.channel = channel;
         this.frameDecoder = frameDecoder;
-        this.references.add(NetworkBytesReference.wrap(new BytesArray(new byte[DEFAULT_READ_LENGTH])));
     }
 
     @Override
     public int read() throws IOException {
-        NetworkBytesReference last = references.peekLast();
-        if (last == null || last.hasWriteRemaining() == false) {
-            this.references.add(NetworkBytesReference.wrap(new BytesArray(new byte[DEFAULT_READ_LENGTH])));
+        if (channelBuffer.getRemaining() == 0) {
+            // Requiring one additional byte will ensure that a new page is allocated.
+            channelBuffer.ensureCapacity(channelBuffer.getCapacity() + 1);
         }
 
-        int bytesRead = channel.read(references.getLast());
+        int bytesRead = channel.read(channelBuffer);
 
         if (bytesRead == -1) {
             return bytesRead;
         }
 
-        rawBytesCount += bytesRead;
-
         BytesReference message;
 
         // Frame decoder will throw an exception if the message is improperly formatted, the header is incorrect,
         // or the message is corrupted
-        while ((message = frameDecoder.decode(createCompositeBuffer(), rawBytesCount)) != null) {
+        while ((message = frameDecoder.decode(toBytesReference(channelBuffer))) != null) {
             int messageLengthWithHeader = message.length();
-            NetworkBytesReference.vectorizedIncrementReadIndexes(references, messageLengthWithHeader);
-            trimDecodedMessages(messageLengthWithHeader);
-            rawBytesCount -= messageLengthWithHeader;
 
             try {
                 BytesReference messageWithoutHeader = message.slice(6, message.length() - 6);
@@ -84,32 +74,22 @@ public class TcpReadContext implements ReadContext {
                 }
             } catch (Exception e) {
                 handler.handleException(channel, e);
+            } finally {
+                channelBuffer.release(messageLengthWithHeader);
             }
         }
 
         return bytesRead;
     }
 
-    private CompositeBytesReference createCompositeBuffer() {
-        return new CompositeBytesReference(references.toArray(new BytesReference[references.size()]));
-    }
-
-    private void trimDecodedMessages(int bytesToTrim) {
-        while (bytesToTrim != 0) {
-            NetworkBytesReference ref = references.getFirst();
-            int readIndex = ref.getReadIndex();
-            bytesToTrim -= readIndex;
-            if (readIndex == ref.length()) {
-                references.removeFirst();
-            } else {
-                assert bytesToTrim == 0;
-                if (readIndex != 0) {
-                    references.removeFirst();
-                    NetworkBytesReference slicedRef = ref.slice(readIndex, ref.length() - readIndex);
-                    references.addFirst(slicedRef);
-                }
-            }
-
+    private static BytesReference toBytesReference(InboundChannelBuffer channelBuffer) {
+        ByteBuffer[] writtenToBuffers = channelBuffer.sliceBuffersTo(channelBuffer.getIndex());
+        ByteBufferReference[] references = new ByteBufferReference[writtenToBuffers.length];
+        for (int i = 0; i < references.length; ++i) {
+            references[i] = new ByteBufferReference(writtenToBuffers[i]);
         }
+
+        return new CompositeBytesReference(references);
     }
+
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/ByteBufferReferenceTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/ByteBufferReferenceTests.java
deleted file mode 100644
index 335e3d2f778..00000000000
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/ByteBufferReferenceTests.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.nio;
-
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.test.ESTestCase;
-
-import java.nio.ByteBuffer;
-
-public class ByteBufferReferenceTests extends ESTestCase {
-
-    private NetworkBytesReference buffer;
-
-    public void testBasicGetByte() {
-        byte[] bytes = new byte[10];
-        initializeBytes(bytes);
-        buffer = NetworkBytesReference.wrap(new BytesArray(bytes));
-
-        assertEquals(10, buffer.length());
-        for (int i = 0 ; i < bytes.length; ++i) {
-            assertEquals(i, buffer.get(i));
-        }
-    }
-
-    public void testBasicGetByteWithOffset() {
-        byte[] bytes = new byte[10];
-        initializeBytes(bytes);
-        buffer = NetworkBytesReference.wrap(new BytesArray(bytes, 2, 8));
-
-        assertEquals(8, buffer.length());
-        for (int i = 2 ; i < bytes.length; ++i) {
-            assertEquals(i, buffer.get(i - 2));
-        }
-    }
-
-    public void testBasicGetByteWithOffsetAndLimit() {
-        byte[] bytes = new byte[10];
-        initializeBytes(bytes);
-        buffer = NetworkBytesReference.wrap(new BytesArray(bytes, 2, 6));
-
-        assertEquals(6, buffer.length());
-        for (int i = 2 ; i < bytes.length - 2; ++i) {
-            assertEquals(i, buffer.get(i - 2));
-        }
-    }
-
-    public void testGetWriteBufferRespectsWriteIndex() {
-        byte[] bytes = new byte[10];
-
-        buffer = NetworkBytesReference.wrap(new BytesArray(bytes, 2, 8));
-
-        ByteBuffer writeByteBuffer = buffer.getWriteByteBuffer();
-
-        assertEquals(2, writeByteBuffer.position());
-        assertEquals(10, writeByteBuffer.limit());
-
-        buffer.incrementWrite(2);
-
-        writeByteBuffer = buffer.getWriteByteBuffer();
-        assertEquals(4, writeByteBuffer.position());
-        assertEquals(10, writeByteBuffer.limit());
-    }
-
-    public void testGetReadBufferRespectsReadIndex() {
-        byte[] bytes = new byte[10];
-
-        buffer = NetworkBytesReference.wrap(new BytesArray(bytes, 3, 6), 6, 0);
-
-        ByteBuffer readByteBuffer = buffer.getReadByteBuffer();
-
-        assertEquals(3, readByteBuffer.position());
-        assertEquals(9, readByteBuffer.limit());
-
-        buffer.incrementRead(2);
-
-        readByteBuffer = buffer.getReadByteBuffer();
-        assertEquals(5, readByteBuffer.position());
-        assertEquals(9, readByteBuffer.limit());
-    }
-
-    public void testWriteAndReadRemaining() {
-        byte[] bytes = new byte[10];
-
-        buffer = NetworkBytesReference.wrap(new BytesArray(bytes, 2, 8));
-
-        assertEquals(0, buffer.getReadRemaining());
-        assertEquals(8, buffer.getWriteRemaining());
-
-        buffer.incrementWrite(3);
-        buffer.incrementRead(2);
-
-        assertEquals(1, buffer.getReadRemaining());
-        assertEquals(5, buffer.getWriteRemaining());
-    }
-
-    public void testBasicSlice() {
-        byte[] bytes = new byte[20];
-        initializeBytes(bytes);
-
-        buffer = NetworkBytesReference.wrap(new BytesArray(bytes, 2, 18));
-
-        NetworkBytesReference slice = buffer.slice(4, 14);
-
-        assertEquals(14, slice.length());
-        assertEquals(0, slice.getReadIndex());
-        assertEquals(0, slice.getWriteIndex());
-
-        for (int i = 6; i < 20; ++i) {
-            assertEquals(i, slice.get(i - 6));
-        }
-    }
-
-    public void testSliceWithReadAndWriteIndexes() {
-        byte[] bytes = new byte[20];
-        initializeBytes(bytes);
-
-        buffer = NetworkBytesReference.wrap(new BytesArray(bytes, 2, 18));
-
-        buffer.incrementWrite(9);
-        buffer.incrementRead(5);
-
-        NetworkBytesReference slice = buffer.slice(6, 12);
-
-        assertEquals(12, slice.length());
-        assertEquals(0, slice.getReadIndex());
-        assertEquals(3, slice.getWriteIndex());
-
-        for (int i = 8; i < 20; ++i) {
-            assertEquals(i, slice.get(i - 8));
-        }
-    }
-
-    private void initializeBytes(byte[] bytes) {
-        for (int i = 0 ; i < bytes.length; ++i) {
-            bytes[i] = (byte) i;
-        }
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/InboundChannelBufferTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/InboundChannelBufferTests.java
new file mode 100644
index 00000000000..7232a938710
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/InboundChannelBufferTests.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport.nio;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.nio.ByteBuffer;
+
+public class InboundChannelBufferTests extends ESTestCase {
+
+    private static final int PAGE_SIZE = 1 << 14;
+
+    public void testNewBufferHasSinglePage() {
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+
+        assertEquals(PAGE_SIZE, channelBuffer.getCapacity());
+        assertEquals(PAGE_SIZE, channelBuffer.getRemaining());
+        assertEquals(0, channelBuffer.getIndex());
+    }
+
+    public void testExpandCapacity() {
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+
+        assertEquals(PAGE_SIZE, channelBuffer.getCapacity());
+        assertEquals(PAGE_SIZE, channelBuffer.getRemaining());
+
+        channelBuffer.ensureCapacity(PAGE_SIZE + 1);
+
+        assertEquals(PAGE_SIZE * 2, channelBuffer.getCapacity());
+        assertEquals(PAGE_SIZE * 2, channelBuffer.getRemaining());
+    }
+
+    public void testExpandCapacityMultiplePages() {
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+
+        assertEquals(PAGE_SIZE, channelBuffer.getCapacity());
+
+        int multiple = randomInt(80);
+        channelBuffer.ensureCapacity(PAGE_SIZE + ((multiple * PAGE_SIZE) - randomInt(500)));
+
+        assertEquals(PAGE_SIZE * (multiple + 1), channelBuffer.getCapacity());
+        assertEquals(PAGE_SIZE * (multiple + 1), channelBuffer.getRemaining());
+    }
+
+    public void testExpandCapacityRespectsOffset() {
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+
+        assertEquals(PAGE_SIZE, channelBuffer.getCapacity());
+        assertEquals(PAGE_SIZE, channelBuffer.getRemaining());
+
+        int offset = randomInt(300);
+
+        channelBuffer.release(offset);
+
+        assertEquals(PAGE_SIZE - offset, channelBuffer.getCapacity());
+        assertEquals(PAGE_SIZE - offset, channelBuffer.getRemaining());
+
+        channelBuffer.ensureCapacity(PAGE_SIZE + 1);
+
+        assertEquals(PAGE_SIZE * 2 - offset, channelBuffer.getCapacity());
+        assertEquals(PAGE_SIZE * 2 - offset, channelBuffer.getRemaining());
+    }
+
+    public void testIncrementIndex() {
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+
+        assertEquals(0, channelBuffer.getIndex());
+        assertEquals(PAGE_SIZE, channelBuffer.getRemaining());
+
+        channelBuffer.incrementIndex(10);
+
+        assertEquals(10, channelBuffer.getIndex());
+        assertEquals(PAGE_SIZE - 10, channelBuffer.getRemaining());
+    }
+
+    public void testIncrementIndexWithOffset() {
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+
+        assertEquals(0, channelBuffer.getIndex());
+        assertEquals(PAGE_SIZE, channelBuffer.getRemaining());
+
+        channelBuffer.release(10);
+        assertEquals(PAGE_SIZE - 10, channelBuffer.getRemaining());
+
+        channelBuffer.incrementIndex(10);
+
+        assertEquals(10, channelBuffer.getIndex());
+        assertEquals(PAGE_SIZE - 20, channelBuffer.getRemaining());
+
+        channelBuffer.release(2);
+        assertEquals(8, channelBuffer.getIndex());
+        assertEquals(PAGE_SIZE - 20, channelBuffer.getRemaining());
+    }
+
+    public void testAccessByteBuffers() {
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+
+        int pages = randomInt(50) + 5;
+        channelBuffer.ensureCapacity(pages * PAGE_SIZE);
+
+        long capacity = channelBuffer.getCapacity();
+
+        ByteBuffer[] postIndexBuffers = channelBuffer.sliceBuffersFrom(channelBuffer.getIndex());
+        int i = 0;
+        for (ByteBuffer buffer : postIndexBuffers) {
+            while (buffer.hasRemaining()) {
+                buffer.put((byte) (i++ % 127));
+            }
+        }
+
+        int indexIncremented = 0;
+        int bytesReleased = 0;
+        while (indexIncremented < capacity) {
+            assertEquals(indexIncremented - bytesReleased, channelBuffer.getIndex());
+
+            long amountToInc = Math.min(randomInt(2000), channelBuffer.getRemaining());
+            ByteBuffer[] postIndexBuffers2 = channelBuffer.sliceBuffersFrom(channelBuffer.getIndex());
+            assertEquals((byte) ((channelBuffer.getIndex() + bytesReleased) % 127), postIndexBuffers2[0].get());
+            ByteBuffer[] preIndexBuffers = channelBuffer.sliceBuffersTo(channelBuffer.getIndex());
+            if (preIndexBuffers.length > 0) {
+                ByteBuffer preIndexBuffer = preIndexBuffers[preIndexBuffers.length - 1];
+                assertEquals((byte) ((channelBuffer.getIndex() + bytesReleased - 1) % 127), preIndexBuffer.get(preIndexBuffer.limit() - 1));
+            }
+            if (randomBoolean()) {
+                long bytesToRelease = Math.min(randomInt(50), channelBuffer.getIndex());
+                channelBuffer.release(bytesToRelease);
+                bytesReleased += bytesToRelease;
+            }
+            channelBuffer.incrementIndex(amountToInc);
+            indexIncremented += amountToInc;
+        }
+
+        assertEquals(0, channelBuffer.sliceBuffersFrom(channelBuffer.getIndex()).length);
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketEventHandlerTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketEventHandlerTests.java
index 8f270d11e5a..b547273d309 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketEventHandlerTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketEventHandlerTests.java
@@ -122,8 +122,7 @@ public class SocketEventHandlerTests extends ESTestCase {
         assertEquals(SelectionKey.OP_READ | SelectionKey.OP_WRITE, selectionKey.interestOps());
 
         BytesArray bytesArray = new BytesArray(new byte[1]);
-        NetworkBytesReference networkBuffer = NetworkBytesReference.wrap(bytesArray);
-        channel.getWriteContext().queueWriteOperations(new WriteOperation(channel, networkBuffer, mock(ActionListener.class)));
+        channel.getWriteContext().queueWriteOperations(new WriteOperation(channel, bytesArray, mock(ActionListener.class)));
 
         when(rawChannel.write(ByteBuffer.wrap(bytesArray.array()))).thenReturn(1);
         handler.handleWrite(channel);
@@ -138,8 +137,7 @@ public class SocketEventHandlerTests extends ESTestCase {
         assertEquals(SelectionKey.OP_READ | SelectionKey.OP_WRITE, selectionKey.interestOps());
 
         BytesArray bytesArray = new BytesArray(new byte[1]);
-        NetworkBytesReference networkBuffer = NetworkBytesReference.wrap(bytesArray, 1, 0);
-        channel.getWriteContext().queueWriteOperations(new WriteOperation(channel, networkBuffer, mock(ActionListener.class)));
+        channel.getWriteContext().queueWriteOperations(new WriteOperation(channel, bytesArray, mock(ActionListener.class)));
 
         when(rawChannel.write(ByteBuffer.wrap(bytesArray.array()))).thenReturn(0);
         handler.handleWrite(channel);
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketSelectorTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketSelectorTests.java
index 34a44ee4e4b..1b67d9d099b 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketSelectorTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/SocketSelectorTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.transport.nio;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.nio.channel.NioSocketChannel;
 import org.elasticsearch.transport.nio.channel.WriteContext;
@@ -53,7 +54,7 @@ public class SocketSelectorTests extends ESTestCase {
     private TestSelectionKey selectionKey;
     private WriteContext writeContext;
     private ActionListener listener;
-    private NetworkBytesReference bufferReference = NetworkBytesReference.wrap(new BytesArray(new byte[1]));
+    private BytesReference bufferReference = new BytesArray(new byte[1]);
     private Selector rawSelector;
 
     @Before
@@ -294,8 +295,7 @@ public class SocketSelectorTests extends ESTestCase {
 
         socketSelector.preSelect();
 
-        NetworkBytesReference networkBuffer = NetworkBytesReference.wrap(new BytesArray(new byte[1]));
-        socketSelector.queueWrite(new WriteOperation(mock(NioSocketChannel.class), networkBuffer, listener));
+        socketSelector.queueWrite(new WriteOperation(mock(NioSocketChannel.class), new BytesArray(new byte[1]), listener));
         socketSelector.scheduleForRegistration(unRegisteredChannel);
 
         TestSelectionKey testSelectionKey = new TestSelectionKey(0);
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/WriteOperationTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/WriteOperationTests.java
index 351ac87eb56..0015d39a373 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/WriteOperationTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/WriteOperationTests.java
@@ -26,6 +26,7 @@ import org.elasticsearch.transport.nio.channel.NioSocketChannel;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
@@ -48,11 +49,7 @@ public class WriteOperationTests extends ESTestCase {
         WriteOperation writeOp = new WriteOperation(channel, new BytesArray(new byte[10]), listener);
 
 
-        when(channel.write(any())).thenAnswer(invocationOnMock -> {
-            NetworkBytesReference[] refs = (NetworkBytesReference[]) invocationOnMock.getArguments()[0];
-            refs[0].incrementRead(10);
-            return 10;
-        });
+        when(channel.write(any(ByteBuffer[].class))).thenReturn(10);
 
         writeOp.flush();
 
@@ -62,15 +59,10 @@ public class WriteOperationTests extends ESTestCase {
     public void testPartialFlush() throws IOException {
         WriteOperation writeOp = new WriteOperation(channel, new BytesArray(new byte[10]), listener);
 
-        when(channel.write(any())).thenAnswer(invocationOnMock -> {
-            NetworkBytesReference[] refs = (NetworkBytesReference[]) invocationOnMock.getArguments()[0];
-            refs[0].incrementRead(5);
-            return 5;
-        });
+        when(channel.write(any(ByteBuffer[].class))).thenReturn(5);
 
         writeOp.flush();
 
         assertFalse(writeOp.isFullyFlushed());
-        assertEquals(5, writeOp.getByteReferences()[0].getReadRemaining());
     }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpFrameDecoderTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpFrameDecoderTests.java
index 519828592be..450016b1dc3 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpFrameDecoderTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpFrameDecoderTests.java
@@ -43,10 +43,8 @@ public class TcpFrameDecoderTests extends ESTestCase {
         streamOutput.write('S');
         streamOutput.write(1);
         streamOutput.write(1);
-        streamOutput.write(0);
-        streamOutput.write(0);
 
-        assertNull(frameDecoder.decode(streamOutput.bytes(), 4));
+        assertNull(frameDecoder.decode(streamOutput.bytes()));
         assertEquals(-1, frameDecoder.expectedMessageLength());
     }
 
@@ -56,7 +54,7 @@ public class TcpFrameDecoderTests extends ESTestCase {
         streamOutput.write('S');
         streamOutput.writeInt(-1);
 
-        BytesReference message = frameDecoder.decode(streamOutput.bytes(), 6);
+        BytesReference message = frameDecoder.decode(streamOutput.bytes());
 
         assertEquals(-1, frameDecoder.expectedMessageLength());
         assertEquals(streamOutput.bytes(), message);
@@ -70,7 +68,7 @@ public class TcpFrameDecoderTests extends ESTestCase {
         streamOutput.write('E');
         streamOutput.write('S');
 
-        BytesReference message = frameDecoder.decode(streamOutput.bytes(), 8);
+        BytesReference message = frameDecoder.decode(streamOutput.bytes());
 
         assertEquals(6, message.length());
         assertEquals(streamOutput.bytes().slice(0, 6), message);
@@ -84,7 +82,7 @@ public class TcpFrameDecoderTests extends ESTestCase {
         streamOutput.write('M');
         streamOutput.write('A');
 
-        BytesReference message = frameDecoder.decode(streamOutput.bytes(), 8);
+        BytesReference message = frameDecoder.decode(streamOutput.bytes());
 
         assertEquals(-1, frameDecoder.expectedMessageLength());
         assertEquals(streamOutput.bytes(), message);
@@ -98,7 +96,7 @@ public class TcpFrameDecoderTests extends ESTestCase {
         streamOutput.write('M');
         streamOutput.write('A');
 
-        BytesReference message = frameDecoder.decode(streamOutput.bytes(), 8);
+        BytesReference message = frameDecoder.decode(streamOutput.bytes());
 
         assertEquals(9, frameDecoder.expectedMessageLength());
         assertNull(message);
@@ -113,7 +111,7 @@ public class TcpFrameDecoderTests extends ESTestCase {
         streamOutput.write('A');
 
         try {
-            frameDecoder.decode(streamOutput.bytes(), 8);
+            frameDecoder.decode(streamOutput.bytes());
             fail("Expected exception");
         } catch (Exception ex) {
             assertThat(ex, instanceOf(StreamCorruptedException.class));
@@ -134,7 +132,7 @@ public class TcpFrameDecoderTests extends ESTestCase {
         streamOutput.write(randomByte());
 
         try {
-            frameDecoder.decode(streamOutput.bytes(), 7);
+            frameDecoder.decode(streamOutput.bytes());
             fail("Expected exception");
         } catch (Exception ex) {
             assertThat(ex, instanceOf(StreamCorruptedException.class));
@@ -158,7 +156,7 @@ public class TcpFrameDecoderTests extends ESTestCase {
 
             try {
                 BytesReference bytes = streamOutput.bytes();
-                frameDecoder.decode(bytes, bytes.length());
+                frameDecoder.decode(bytes);
                 fail("Expected exception");
             } catch (Exception ex) {
                 assertThat(ex, instanceOf(TcpTransport.HttpOnTransportException.class));
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpReadContextTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpReadContextTests.java
index 7586b5abd91..73583353f73 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpReadContextTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpReadContextTests.java
@@ -22,13 +22,13 @@ package org.elasticsearch.transport.nio.channel;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.CompositeBytesReference;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.transport.nio.NetworkBytesReference;
+import org.elasticsearch.transport.nio.InboundChannelBuffer;
 import org.elasticsearch.transport.nio.TcpReadHandler;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.mockito.Matchers.any;
@@ -57,13 +57,13 @@ public class TcpReadContextTests extends ESTestCase {
         byte[] bytes = createMessage(messageLength);
         byte[] fullMessage = combineMessageAndHeader(bytes);
 
-        final AtomicInteger bufferCapacity = new AtomicInteger();
-        when(channel.read(any(NetworkBytesReference.class))).thenAnswer(invocationOnMock -> {
-            NetworkBytesReference reference = (NetworkBytesReference) invocationOnMock.getArguments()[0];
-            ByteBuffer buffer = reference.getWriteByteBuffer();
-            bufferCapacity.set(reference.getWriteRemaining());
-            buffer.put(fullMessage);
-            reference.incrementWrite(fullMessage.length);
+        final AtomicLong bufferCapacity = new AtomicLong();
+        when(channel.read(any(InboundChannelBuffer.class))).thenAnswer(invocationOnMock -> {
+            InboundChannelBuffer buffer = (InboundChannelBuffer) invocationOnMock.getArguments()[0];
+            ByteBuffer byteBuffer = buffer.sliceBuffersFrom(buffer.getIndex())[0];
+            bufferCapacity.set(buffer.getCapacity() - buffer.getIndex());
+            byteBuffer.put(fullMessage);
+            buffer.incrementIndex(fullMessage.length);
             return fullMessage.length;
         });
 
@@ -82,15 +82,15 @@ public class TcpReadContextTests extends ESTestCase {
         byte[] fullPart1 = combineMessageAndHeader(part1, messageLength + messageLength);
         byte[] part2 = createMessage(messageLength);
 
-        final AtomicInteger bufferCapacity = new AtomicInteger();
+        final AtomicLong bufferCapacity = new AtomicLong();
         final AtomicReference bytes = new AtomicReference<>();
 
-        when(channel.read(any(NetworkBytesReference.class))).thenAnswer(invocationOnMock -> {
-            NetworkBytesReference reference = (NetworkBytesReference) invocationOnMock.getArguments()[0];
-            ByteBuffer buffer = reference.getWriteByteBuffer();
-            bufferCapacity.set(reference.getWriteRemaining());
-            buffer.put(bytes.get());
-            reference.incrementWrite(bytes.get().length);
+        when(channel.read(any(InboundChannelBuffer.class))).thenAnswer(invocationOnMock -> {
+            InboundChannelBuffer buffer = (InboundChannelBuffer) invocationOnMock.getArguments()[0];
+            ByteBuffer byteBuffer = buffer.sliceBuffersFrom(buffer.getIndex())[0];
+            bufferCapacity.set(buffer.getCapacity() - buffer.getIndex());
+            byteBuffer.put(bytes.get());
+            buffer.incrementIndex(bytes.get().length);
             return bytes.get().length;
         });
 
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpWriteContextTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpWriteContextTests.java
index 16b53cd71b0..33b84590aaa 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpWriteContextTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpWriteContextTests.java
@@ -80,7 +80,7 @@ public class TcpWriteContextTests extends ESTestCase {
 
         assertSame(listener, writeOp.getListener());
         assertSame(channel, writeOp.getChannel());
-        assertEquals(ByteBuffer.wrap(bytes), writeOp.getByteReferences()[0].getReadByteBuffer());
+        assertEquals(ByteBuffer.wrap(bytes), writeOp.getByteBuffers()[0]);
     }
 
     public void testSendMessageFromSameThreadIsQueuedInChannel() throws Exception {
@@ -97,7 +97,7 @@ public class TcpWriteContextTests extends ESTestCase {
 
         assertSame(listener, writeOp.getListener());
         assertSame(channel, writeOp.getChannel());
-        assertEquals(ByteBuffer.wrap(bytes), writeOp.getByteReferences()[0].getReadByteBuffer());
+        assertEquals(ByteBuffer.wrap(bytes), writeOp.getByteBuffers()[0]);
     }
 
     public void testWriteIsQueuedInChannel() throws Exception {

From 6c7374804ff1bebaa4fdf93ceb0f8c3a3a068a78 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Wed, 6 Dec 2017 18:03:13 -0500
Subject: [PATCH 199/297] Extend JVM options to support multiple versions

JDK 9 has removed JVM options that were valid in JDK 8 (e.g., GC logging
flags) and replaced them with new flags that are not available in JDK
8. This means that a single JVM options file can no longer apply to JDK
8 and JDK 9, complicating development, complicating our packaging story,
and complicating operations. This commit extends the JVM options syntax
to specify the range of versions the option applies to. If the running
JVM matches the range of versions, the flag will be used to start the
JVM otherwise the flag will be ignored.

We implement this parser in Java for simplicity, and with this we start
our first step towards a Java launcher.

Relates #27675
---
 distribution/build.gradle                     |   2 +-
 .../src/main/resources/bin/elasticsearch      |   9 +-
 .../src/main/resources/bin/elasticsearch-env  |   2 +-
 .../main/resources/bin/elasticsearch-env.bat  |   2 +-
 .../resources/bin/elasticsearch-service.bat   |  12 +-
 .../src/main/resources/bin/elasticsearch.bat  |  11 +-
 .../src/main/resources/config/jvm.options     |  24 +-
 .../build.gradle                              |  24 +-
 .../tools/launchers/JavaVersion.java}         |  61 ++--
 .../tools/launchers/JavaVersionChecker.java   |  54 ++++
 .../tools/launchers/JvmOptionsParser.java     | 270 ++++++++++++++++++
 .../tools/launchers/Launchers.java            |  57 ++++
 .../tools/launchers}/SuppressForbidden.java   |   2 +-
 .../launchers/JvmOptionsParserTests.java      | 248 ++++++++++++++++
 .../tools/launchers/LaunchersTestCase.java    |  45 +++
 .../setup/sysconfig/configuring.asciidoc      |  21 +-
 settings.gradle                               |   2 +-
 17 files changed, 761 insertions(+), 85 deletions(-)
 rename distribution/tools/{java-version-checker => launchers}/build.gradle (61%)
 rename distribution/tools/{java-version-checker/src/main/java/org/elasticsearch/tools/JavaVersionChecker.java => launchers/src/main/java/org/elasticsearch/tools/launchers/JavaVersion.java} (50%)
 create mode 100644 distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JavaVersionChecker.java
 create mode 100644 distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java
 create mode 100644 distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/Launchers.java
 rename distribution/tools/{java-version-checker/src/main/java/org/elasticsearch/tools => launchers/src/main/java/org/elasticsearch/tools/launchers}/SuppressForbidden.java (96%)
 create mode 100644 distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmOptionsParserTests.java
 create mode 100644 distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/LaunchersTestCase.java

diff --git a/distribution/build.gradle b/distribution/build.gradle
index a9298718a39..3df81d24c6b 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -164,7 +164,7 @@ configure(distributions) {
       from project(':core').jar
       from project(':core').configurations.runtime
       // delay add tools using closures, since they have not yet been configured, so no jar task exists yet
-      from { project(':distribution:tools:java-version-checker').jar }
+      from { project(':distribution:tools:launchers').jar }
       from { project(':distribution:tools:plugin-cli').jar }
     }
 
diff --git a/distribution/src/main/resources/bin/elasticsearch b/distribution/src/main/resources/bin/elasticsearch
index 4064170807f..11efddf6e26 100755
--- a/distribution/src/main/resources/bin/elasticsearch
+++ b/distribution/src/main/resources/bin/elasticsearch
@@ -16,15 +16,8 @@
 
 source "`dirname "$0"`"/elasticsearch-env
 
-parse_jvm_options() {
-  if [ -f "$1" ]; then
-    echo "`grep "^-" "$1" | tr '\n' ' '`"
-  fi
-}
-
 ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options
-
-JVM_OPTIONS=`parse_jvm_options "$ES_JVM_OPTIONS"`
+JVM_OPTIONS=`"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_JVM_OPTIONS"`
 ES_JAVA_OPTS="${JVM_OPTIONS//\$\{ES_TMPDIR\}/$ES_TMPDIR} $ES_JAVA_OPTS"
 
 cd "$ES_HOME"
diff --git a/distribution/src/main/resources/bin/elasticsearch-env b/distribution/src/main/resources/bin/elasticsearch-env
index 487dcf20922..83380400173 100644
--- a/distribution/src/main/resources/bin/elasticsearch-env
+++ b/distribution/src/main/resources/bin/elasticsearch-env
@@ -63,7 +63,7 @@ if [ ! -z "$JAVA_OPTS" ]; then
 fi
 
 # check the Java version
-"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.JavaVersionChecker
+"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JavaVersionChecker
 
 export HOSTNAME=$HOSTNAME
 
diff --git a/distribution/src/main/resources/bin/elasticsearch-env.bat b/distribution/src/main/resources/bin/elasticsearch-env.bat
index e80a8ce258e..4d1ea24b388 100644
--- a/distribution/src/main/resources/bin/elasticsearch-env.bat
+++ b/distribution/src/main/resources/bin/elasticsearch-env.bat
@@ -42,7 +42,7 @@ if defined JAVA_OPTS (
 )
 
 rem check the Java version
-%JAVA% -cp "%ES_CLASSPATH%" "org.elasticsearch.tools.JavaVersionChecker" || exit /b 1
+%JAVA% -cp "%ES_CLASSPATH%" "org.elasticsearch.tools.launchers.JavaVersionChecker" || exit /b 1
 
 set HOSTNAME=%COMPUTERNAME%
 
diff --git a/distribution/src/main/resources/bin/elasticsearch-service.bat b/distribution/src/main/resources/bin/elasticsearch-service.bat
index e8be9485349..6218d120627 100644
--- a/distribution/src/main/resources/bin/elasticsearch-service.bat
+++ b/distribution/src/main/resources/bin/elasticsearch-service.bat
@@ -103,11 +103,19 @@ set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
 if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
 
 @setlocal
-for /F "usebackq delims=" %%a in (`findstr /b \- "%ES_JVM_OPTIONS%" ^| findstr /b /v "\-server \-client"`) do set JVM_OPTIONS=!JVM_OPTIONS!%%a;
-@endlocal & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!%%ES_JAVA_OPTS%
+for /F "usebackq delims=" %%a in (`"%JAVA% -cp "%ES_CLASSPATH%" "org.elasticsearch.tools.launchers.JvmOptionsParser" "%ES_JVM_OPTIONS%" || echo jvm_options_parser_failed"`) do set JVM_OPTIONS=%%a
+@endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%JVM_OPTIONS%" & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!% %ES_JAVA_OPTS%
+
+if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
+  exit /b 1
+)
+
+if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
 
 if "%ES_JAVA_OPTS:~-1%"==";" set ES_JAVA_OPTS=%ES_JAVA_OPTS:~0,-1%
 
+echo %ES_JAVA_OPTS%
+
 @setlocal EnableDelayedExpansion
 for %%a in ("%ES_JAVA_OPTS:;=","%") do (
   set var=%%a
diff --git a/distribution/src/main/resources/bin/elasticsearch.bat b/distribution/src/main/resources/bin/elasticsearch.bat
index 210da3e5eb6..4709942d0dc 100644
--- a/distribution/src/main/resources/bin/elasticsearch.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.bat
@@ -42,12 +42,13 @@ IF ERRORLEVEL 1 (
 )
 
 set "ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options"
-
 @setlocal
-rem extract the options from the JVM options file %ES_JVM_OPTIONS%
-rem such options are the lines beginning with '-', thus "findstr /b"
-for /F "usebackq delims=" %%a in (`findstr /b \- "%ES_JVM_OPTIONS%"`) do set JVM_OPTIONS=!JVM_OPTIONS! %%a
-@endlocal & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!% %ES_JAVA_OPTS%
+for /F "usebackq delims=" %%a in (`"%JAVA% -cp "%ES_CLASSPATH%" "org.elasticsearch.tools.launchers.JvmOptionsParser" "%ES_JVM_OPTIONS%" || echo jvm_options_parser_failed"`) do set JVM_OPTIONS=%%a
+@endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%JVM_OPTIONS%" & set ES_JAVA_OPTS=%JVM_OPTIONS:${ES_TMPDIR}=!ES_TMPDIR!% %ES_JAVA_OPTS%
+
+if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
+  exit /b 1
+)
 
 cd "%ES_HOME%"
 %JAVA% %ES_JAVA_OPTS% -Delasticsearch -Des.path.home="%ES_HOME%" -Des.path.conf="%ES_PATH_CONF%" -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams!
diff --git a/distribution/src/main/resources/config/jvm.options b/distribution/src/main/resources/config/jvm.options
index a8fff81f468..85209c21ca4 100644
--- a/distribution/src/main/resources/config/jvm.options
+++ b/distribution/src/main/resources/config/jvm.options
@@ -44,9 +44,6 @@
 
 ## basic
 
-# force the server VM
--server
-
 # explicitly set the stack size
 -Xss1m
 
@@ -84,13 +81,16 @@
 # ensure the directory exists and has sufficient space
 ${heap.dump.path}
 
-## GC logging
+## JDK 8 GC logging
 
--XX:+PrintGCDetails
--XX:+PrintGCDateStamps
--XX:+PrintTenuringDistribution
--XX:+PrintGCApplicationStoppedTime
--Xloggc:${loggc}
--XX:+UseGCLogFileRotation
--XX:NumberOfGCLogFiles=32
--XX:GCLogFileSize=64m
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:${loggc}
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+# JDK 9+ GC logging
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=${loggc}:utctime,pid,tags:filecount=32,filesize=64m
diff --git a/distribution/tools/java-version-checker/build.gradle b/distribution/tools/launchers/build.gradle
similarity index 61%
rename from distribution/tools/java-version-checker/build.gradle
rename to distribution/tools/launchers/build.gradle
index 7b2a76037cc..27e8712ffcb 100644
--- a/distribution/tools/java-version-checker/build.gradle
+++ b/distribution/tools/launchers/build.gradle
@@ -17,26 +17,38 @@
  * under the License.
  */
 
-
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.JavaVersion
 
 apply plugin: 'elasticsearch.build'
 apply plugin: 'ru.vyarus.animalsniffer'
 
-sourceCompatibility = JavaVersion.VERSION_1_6
-targetCompatibility = JavaVersion.VERSION_1_6
+sourceCompatibility = JavaVersion.VERSION_1_7
+targetCompatibility = JavaVersion.VERSION_1_7
 
 dependencies {
-    signature "org.codehaus.mojo.signature:java16:1.0@signature"
+    signature "org.codehaus.mojo.signature:java17:1.0@signature"
+
+    testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+    testCompile "junit:junit:${versions.junit}"
+    testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
 }
 
+archivesBaseName = 'elasticsearch-launchers'
+
+// launchers do not depend on core so only JDK signatures should be checked
 forbiddenApisMain {
-    // java-version-checker does not depend on core so only JDK signatures should be checked
+    signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}
+forbiddenApisTest {
     signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
+namingConventions {
+    testClass = 'org.elasticsearch.tools.launchers.LaunchersTestCase'
+    skipIntegTestInDisguise = true
+}
+
 javadoc.enabled = false
-test.enabled = false
 loggerUsageCheck.enabled = false
 jarHell.enabled=false
diff --git a/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/JavaVersionChecker.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JavaVersion.java
similarity index 50%
rename from distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/JavaVersionChecker.java
rename to distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JavaVersion.java
index 1b44132c55e..30ca7a4a2a7 100644
--- a/distribution/tools/java-version-checker/src/main/java/org/elasticsearch/tools/JavaVersionChecker.java
+++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JavaVersion.java
@@ -17,47 +17,18 @@
  * under the License.
  */
 
-package org.elasticsearch.tools;
+package org.elasticsearch.tools.launchers;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
-import java.util.Locale;
+import java.util.Objects;
 
-/**
- * Simple program that checks if the runtime Java version is at least 1.8.
- */
-final class JavaVersionChecker {
+public class JavaVersion {
 
-    private JavaVersionChecker() {
-    }
+    static final List CURRENT = parse(System.getProperty("java.specification.version"));
+    static final List JAVA_8 = parse("1.8");
 
-    private static final List JAVA_8 = Arrays.asList(1, 8);
-
-    /**
-     * The main entry point. The exit code is 0 if the Java version is at least 1.8, otherwise the exit code is 1.
-     *
-     * @param args the args to the program which are rejected if not empty
-     */
-    public static void main(final String[] args) {
-        // no leniency!
-        if (args.length != 0) {
-            throw new IllegalArgumentException("expected zero arguments but was: " + Arrays.toString(args));
-        }
-        final String javaSpecificationVersion = System.getProperty("java.specification.version");
-        final List current = parse(javaSpecificationVersion);
-        if (compare(current, JAVA_8) < 0) {
-            final String message = String.format(
-                    Locale.ROOT,
-                    "the minimum required Java version is 8; your Java version from [%s] does not meet this requirement",
-                    System.getProperty("java.home"));
-            println(message);
-            exit(1);
-        }
-        exit(0);
-    }
-
-    private static List parse(final String value) {
+    static List parse(final String value) {
         if (!value.matches("^0*[0-9]+(\\.[0-9]+)*$")) {
             throw new IllegalArgumentException(value);
         }
@@ -70,7 +41,16 @@ final class JavaVersionChecker {
         return version;
     }
 
-    private static int compare(final List left, final List right) {
+    static int majorVersion(final List javaVersion) {
+        Objects.requireNonNull(javaVersion);
+        if (javaVersion.get(0) > 1) {
+            return javaVersion.get(0);
+        } else {
+            return javaVersion.get(1);
+        }
+    }
+
+    static int compare(final List left, final List right) {
         // lexicographically compare two lists, treating missing entries as zeros
         final int len = Math.max(left.size(), right.size());
         for (int i = 0; i < len; i++) {
@@ -86,14 +66,5 @@ final class JavaVersionChecker {
         return 0;
     }
 
-    @SuppressForbidden(reason = "System#err")
-    private static void println(String message) {
-        System.err.println(message);
-    }
-
-    @SuppressForbidden(reason = "System#exit")
-    private static void exit(final int status) {
-        System.exit(status);
-    }
 
 }
diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JavaVersionChecker.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JavaVersionChecker.java
new file mode 100644
index 00000000000..ed632d060a5
--- /dev/null
+++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JavaVersionChecker.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import java.util.Arrays;
+import java.util.Locale;
+
+/**
+ * Simple program that checks if the runtime Java version is at least 1.8.
+ */
+final class JavaVersionChecker {
+
+    private JavaVersionChecker() {
+    }
+
+    /**
+     * The main entry point. The exit code is 0 if the Java version is at least 1.8, otherwise the exit code is 1.
+     *
+     * @param args the args to the program which are rejected if not empty
+     */
+    public static void main(final String[] args) {
+        // no leniency!
+        if (args.length != 0) {
+            throw new IllegalArgumentException("expected zero arguments but was " + Arrays.toString(args));
+        }
+        if (JavaVersion.compare(JavaVersion.CURRENT, JavaVersion.JAVA_8) < 0) {
+            final String message = String.format(
+                    Locale.ROOT,
+                    "the minimum required Java version is 8; your Java version from [%s] does not meet this requirement",
+                    System.getProperty("java.home"));
+            Launchers.errPrintln(message);
+            Launchers.exit(1);
+        }
+        Launchers.exit(0);
+    }
+
+}
diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java
new file mode 100644
index 00000000000..fe7e045e6bc
--- /dev/null
+++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Parses JVM options from a file and prints a single line with all JVM options to standard output.
+ */
+final class JvmOptionsParser {
+
+    /**
+     * The main entry point. The exit code is 0 if the JVM options were successfully parsed, otherwise the exit code is 1. If an improperly
+     * formatted line is discovered, the line is output to standard error.
+     *
+     * @param args the args to the program which should consist of a single option, the path to the JVM options
+     */
+    public static void main(final String[] args) throws IOException {
+        if (args.length != 1) {
+            throw new IllegalArgumentException("expected one argument specifying path to jvm.options but was " + Arrays.toString(args));
+        }
+        final List jvmOptions = new ArrayList<>();
+        final SortedMap invalidLines = new TreeMap<>();
+        try (InputStream is = Files.newInputStream(Paths.get(args[0]));
+             Reader reader = new InputStreamReader(is, Charset.forName("UTF-8"));
+             BufferedReader br = new BufferedReader(reader)) {
+            parse(
+                    JavaVersion.majorVersion(JavaVersion.CURRENT),
+                    br,
+                    new JvmOptionConsumer() {
+                        @Override
+                        public void accept(final String jvmOption) {
+                            jvmOptions.add(jvmOption);
+                        }
+                    },
+                    new InvalidLineConsumer() {
+                        @Override
+                        public void accept(final int lineNumber, final String line) {
+                            invalidLines.put(lineNumber, line);
+                        }
+                    });
+        }
+
+        if (invalidLines.isEmpty()) {
+            final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(jvmOptions);
+            Launchers.outPrintln(spaceDelimitedJvmOptions);
+            Launchers.exit(0);
+        } else {
+            final String errorMessage = String.format(
+                    Locale.ROOT,
+                    "encountered [%d] error%s parsing [%s]",
+                    invalidLines.size(),
+                    invalidLines.size() == 1 ? "" : "s",
+                    args[0]);
+            Launchers.errPrintln(errorMessage);
+            int count = 0;
+            for (final Map.Entry entry : invalidLines.entrySet()) {
+                count++;
+                final String message = String.format(
+                        Locale.ROOT,
+                        "[%d]: encountered improperly formatted JVM option line [%s] on line number [%d]",
+                        count,
+                        entry.getValue(),
+                        entry.getKey());
+                Launchers.errPrintln(message);
+            }
+            Launchers.exit(1);
+        }
+    }
+
+    /**
+     * Callback for valid JVM options.
+     */
+    interface JvmOptionConsumer {
+        /**
+         * Invoked when a line in the JVM options file matches the specified syntax and the specified major version.
+         * @param jvmOption the matching JVM option
+         */
+        void accept(String jvmOption);
+    }
+
+    /**
+     * Callback for invalid lines in the JVM options.
+     */
+    interface InvalidLineConsumer {
+        /**
+         * Invoked when a line in the JVM options does not match the specified syntax.
+         */
+        void accept(int lineNumber, String line);
+    }
+
+    private static final Pattern PATTERN = Pattern.compile("((?\\d+)(?-)?(?\\d+)?:)?(?
    
  * See Indices API on elastic.co
  */
 public final class IndicesClient {
     private final RestHighLevelClient restHighLevelClient;
 
-    public IndicesClient(RestHighLevelClient restHighLevelClient) {
+    IndicesClient(RestHighLevelClient restHighLevelClient) {
         this.restHighLevelClient = restHighLevelClient;
     }
 
@@ -56,8 +58,32 @@ public final class IndicesClient {
      * See 
      * Delete Index API on elastic.co
      */
-    public void deleteIndexAsync(DeleteIndexRequest deleteIndexRequest, ActionListener listener, Header... headers) {
+    public void deleteIndexAsync(DeleteIndexRequest deleteIndexRequest, ActionListener listener,
+                                       Header... headers) {
         restHighLevelClient.performRequestAsyncAndParseEntity(deleteIndexRequest, Request::deleteIndex, DeleteIndexResponse::fromXContent,
             listener, Collections.emptySet(), headers);
     }
+
+    /**
+     * Creates an index using the Create Index API
+     * 
    
+     * See 
+     * Create Index API on elastic.co
+     */
+    public CreateIndexResponse createIndex(CreateIndexRequest createIndexRequest, Header... headers) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(createIndexRequest, Request::createIndex, CreateIndexResponse::fromXContent,
+            Collections.emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously creates an index using the Create Index API
+     * 
    
+     * See 
+     * Create Index API on elastic.co
+     */
+    public void createIndexAsync(CreateIndexRequest createIndexRequest, ActionListener listener,
+                                       Header... headers) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(createIndexRequest, Request::createIndex, CreateIndexResponse::fromXContent,
+            listener, Collections.emptySet(), headers);
+    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
index 05ce54437a4..a3544ddb89b 100755
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
@@ -29,6 +29,7 @@ import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.entity.ContentType;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -137,6 +138,19 @@ public final class Request {
         return new Request(HttpDelete.METHOD_NAME, endpoint, parameters.getParams(), null);
     }
 
+    static Request createIndex(CreateIndexRequest createIndexRequest) throws IOException {
+        String endpoint = endpoint(createIndexRequest.indices(), Strings.EMPTY_ARRAY, "");
+
+        Params parameters = Params.builder();
+        parameters.withTimeout(createIndexRequest.timeout());
+        parameters.withMasterTimeout(createIndexRequest.masterNodeTimeout());
+        parameters.withWaitForActiveShards(createIndexRequest.waitForActiveShards());
+        parameters.withUpdateAllTypes(createIndexRequest.updateAllTypes());
+
+        HttpEntity entity = createEntity(createIndexRequest, REQUEST_BODY_CONTENT_TYPE);
+        return new Request(HttpPut.METHOD_NAME, endpoint, parameters.getParams(), entity);
+    }
+
     static Request info() {
         return new Request(HttpGet.METHOD_NAME, "/", Collections.emptyMap(), null);
     }
@@ -534,6 +548,13 @@ public final class Request {
             return putParam("timeout", timeout);
         }
 
+        Params withUpdateAllTypes(boolean updateAllTypes) {
+            if (updateAllTypes) {
+                return putParam("update_all_types", Boolean.TRUE.toString());
+            }
+            return this;
+        }
+
         Params withVersion(long version) {
             if (version != Versions.MATCH_ANY) {
                 return putParam("version", Long.toString(version));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
index 4045e565288..0d6430b5912 100755
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
@@ -20,14 +20,88 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 
 public class IndicesClientIT extends ESRestHighLevelClientTestCase {
 
+    @SuppressWarnings("unchecked")
+    public void testCreateIndex() throws IOException {
+        {
+            // Create index
+            String indexName = "plain_index";
+            assertFalse(indexExists(indexName));
+
+            CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
+
+            CreateIndexResponse createIndexResponse =
+                execute(createIndexRequest, highLevelClient().indices()::createIndex, highLevelClient().indices()::createIndexAsync);
+            assertTrue(createIndexResponse.isAcknowledged());
+
+            assertTrue(indexExists(indexName));
+        }
+        {
+            // Create index with mappings, aliases and settings
+            String indexName = "rich_index";
+            assertFalse(indexExists(indexName));
+
+            CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
+
+            Alias alias = new Alias("alias_name");
+            alias.filter("{\"term\":{\"year\":2016}}");
+            alias.routing("1");
+            createIndexRequest.alias(alias);
+
+            Settings.Builder settings = Settings.builder();
+            settings.put(SETTING_NUMBER_OF_REPLICAS, 2);
+            createIndexRequest.settings(settings);
+
+            XContentBuilder mappingBuilder = JsonXContent.contentBuilder();
+            mappingBuilder.startObject().startObject("properties").startObject("field");
+            mappingBuilder.field("type", "text");
+            mappingBuilder.endObject().endObject().endObject();
+            createIndexRequest.mapping("type_name", mappingBuilder);
+
+            CreateIndexResponse createIndexResponse =
+                execute(createIndexRequest, highLevelClient().indices()::createIndex, highLevelClient().indices()::createIndexAsync);
+            assertTrue(createIndexResponse.isAcknowledged());
+
+            Map indexMetaData = getIndexMetadata(indexName);
+
+            Map settingsData = (Map) indexMetaData.get("settings");
+            Map indexSettings = (Map) settingsData.get("index");
+            assertEquals("2", indexSettings.get("number_of_replicas"));
+
+            Map aliasesData = (Map) indexMetaData.get("aliases");
+            Map aliasData = (Map) aliasesData.get("alias_name");
+            assertEquals("1", aliasData.get("index_routing"));
+            Map filter = (Map) aliasData.get("filter");
+            Map term = (Map) filter.get("term");
+            assertEquals(2016, term.get("year"));
+
+            Map mappingsData = (Map) indexMetaData.get("mappings");
+            Map typeData = (Map) mappingsData.get("type_name");
+            Map properties = (Map) typeData.get("properties");
+            Map field = (Map) properties.get("field");
+
+            assertEquals("text", field.get("type"));
+        }
+    }
+
     public void testDeleteIndex() throws IOException {
         {
             // Delete index if exists
@@ -65,4 +139,18 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
 
         return response.getStatusLine().getStatusCode() == 200;
     }
+
+    @SuppressWarnings("unchecked")
+    private Map getIndexMetadata(String index) throws IOException {
+        Response response = client().performRequest("GET", index);
+
+        XContentType entityContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue());
+        Map responseEntity = XContentHelper.convertToMap(entityContentType.xContent(), response.getEntity().getContent(),
+            false);
+
+        Map indexMetaData = (Map) responseEntity.get(index);
+        assertNotNull(indexMetaData);
+
+        return indexMetaData;
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
index f72a7cb4dbf..182de30fd15 100755
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
@@ -25,6 +25,7 @@ import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkShardRequest;
@@ -36,6 +37,7 @@ import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -253,6 +255,34 @@ public class RequestTests extends ESTestCase {
         assertEquals(method, request.getMethod());
     }
 
+    public void testCreateIndex() throws IOException {
+        CreateIndexRequest createIndexRequest = new CreateIndexRequest();
+
+        String indexName = "index-" + randomAlphaOfLengthBetween(2, 5);
+
+        createIndexRequest.index(indexName);
+
+        Map expectedParams = new HashMap<>();
+
+        setRandomTimeout(createIndexRequest::timeout, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams);
+        setRandomMasterTimeout(createIndexRequest, expectedParams);
+        setRandomWaitForActiveShards(createIndexRequest::waitForActiveShards, expectedParams);
+
+        if (randomBoolean()) {
+            boolean updateAllTypes = randomBoolean();
+            createIndexRequest.updateAllTypes(updateAllTypes);
+            if (updateAllTypes) {
+                expectedParams.put("update_all_types", Boolean.TRUE.toString());
+            }
+        }
+
+        Request request = Request.createIndex(createIndexRequest);
+        assertEquals("/" + indexName, request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+        assertEquals("PUT", request.getMethod());
+        assertToXContentBody(createIndexRequest, request.getEntity());
+    }
+
     public void testDeleteIndex() throws IOException {
         DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest();
 
@@ -407,11 +437,7 @@ public class RequestTests extends ESTestCase {
                 expectedParams.put("refresh", refreshPolicy.getValue());
             }
         }
-        if (randomBoolean()) {
-            int waitForActiveShards = randomIntBetween(0, 10);
-            updateRequest.waitForActiveShards(waitForActiveShards);
-            expectedParams.put("wait_for_active_shards", String.valueOf(waitForActiveShards));
-        }
+        setRandomWaitForActiveShards(updateRequest::waitForActiveShards, expectedParams);
         if (randomBoolean()) {
             long version = randomLong();
             updateRequest.version(version);
@@ -1016,6 +1042,14 @@ public class RequestTests extends ESTestCase {
         }
     }
 
+    private static void setRandomWaitForActiveShards(Consumer setter, Map expectedParams) {
+        if (randomBoolean()) {
+            int waitForActiveShards = randomIntBetween(0, 10);
+            setter.accept(waitForActiveShards);
+            expectedParams.put("wait_for_active_shards", String.valueOf(waitForActiveShards));
+        }
+    }
+
     private static void setRandomRefreshPolicy(ReplicatedWriteRequest request, Map expectedParams) {
         if (randomBoolean()) {
             WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
index e866fb92aae..372cc17d137 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
@@ -21,13 +21,18 @@ package org.elasticsearch.client.documentation;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
-import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -52,8 +57,8 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
         RestHighLevelClient client = highLevelClient();
 
         {
-            Response createIndexResponse = client().performRequest("PUT", "/posts");
-            assertEquals(200, createIndexResponse.getStatusLine().getStatusCode());
+            CreateIndexResponse createIndexResponse = client.indices().createIndex(new CreateIndexRequest("posts"));
+            assertTrue(createIndexResponse.isAcknowledged());
         }
 
         {
@@ -61,14 +66,26 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             DeleteIndexRequest request = new DeleteIndexRequest("posts"); // <1>
             // end::delete-index-request
 
+            // tag::delete-index-request-timeout
+            request.timeout(TimeValue.timeValueMinutes(2)); // <1>
+            request.timeout("2m"); // <2>
+            // end::delete-index-request-timeout
+            // tag::delete-index-request-masterTimeout
+            request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1>
+            request.masterNodeTimeout("1m"); // <2>
+            // end::delete-index-request-masterTimeout
+            // tag::delete-index-request-indicesOptions
+            request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1>
+            // end::delete-index-request-indicesOptions
+
             // tag::delete-index-execute
             DeleteIndexResponse deleteIndexResponse = client.indices().deleteIndex(request);
             // end::delete-index-execute
-            assertTrue(deleteIndexResponse.isAcknowledged());
 
             // tag::delete-index-response
             boolean acknowledged = deleteIndexResponse.isAcknowledged(); // <1>
             // end::delete-index-response
+            assertTrue(acknowledged);
 
             // tag::delete-index-execute-async
             client.indices().deleteIndexAsync(request, new ActionListener() {
@@ -85,26 +102,11 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             // end::delete-index-execute-async
         }
 
-        {
-            DeleteIndexRequest request = new DeleteIndexRequest("posts");
-            // tag::delete-index-request-timeout
-            request.timeout(TimeValue.timeValueMinutes(2)); // <1>
-            request.timeout("2m"); // <2>
-            // end::delete-index-request-timeout
-            // tag::delete-index-request-masterTimeout
-            request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1>
-            request.timeout("1m"); // <2>
-            // end::delete-index-request-masterTimeout
-            // tag::delete-index-request-indicesOptions
-            request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1>
-            // end::delete-index-request-indicesOptions
-        }
-
         {
             // tag::delete-index-notfound
             try {
                 DeleteIndexRequest request = new DeleteIndexRequest("does_not_exist");
-                DeleteIndexResponse deleteIndexResponse = client.indices().deleteIndex(request);
+                client.indices().deleteIndex(request);
             } catch (ElasticsearchException exception) {
                 if (exception.status() == RestStatus.NOT_FOUND) {
                     // <1>
@@ -113,4 +115,79 @@ public class IndicesClientDocumentationIT extends ESRestHighLevelClientTestCase
             // end::delete-index-notfound
         }
     }
+
+    public void testCreateIndex() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+
+        {
+            // tag::create-index-request
+            CreateIndexRequest request = new CreateIndexRequest("twitter"); // <1>
+            // end::create-index-request
+
+            // tag::create-index-request-settings
+            request.settings(Settings.builder() // <1>
+                .put("index.number_of_shards", 3)
+                .put("index.number_of_replicas", 2)
+            );
+            // end::create-index-request-settings
+
+            // tag::create-index-request-mappings
+            request.mapping("tweet", // <1>
+                "  {\n" +
+                "    \"tweet\": {\n" +
+                "      \"properties\": {\n" +
+                "        \"message\": {\n" +
+                "          \"type\": \"text\"\n" +
+                "        }\n" +
+                "      }\n" +
+                "    }\n" +
+                "  }", // <2>
+                XContentType.JSON);
+            // end::create-index-request-mappings
+
+            // tag::create-index-request-aliases
+            request.alias(
+                new Alias("twitter_alias")  // <1>
+            );
+            // end::create-index-request-aliases
+
+            // tag::create-index-request-timeout
+            request.timeout(TimeValue.timeValueMinutes(2)); // <1>
+            request.timeout("2m"); // <2>
+            // end::create-index-request-timeout
+            // tag::create-index-request-masterTimeout
+            request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1>
+            request.masterNodeTimeout("1m"); // <2>
+            // end::create-index-request-masterTimeout
+            // tag::create-index-request-waitForActiveShards
+            request.waitForActiveShards(2); // <1>
+            request.waitForActiveShards(ActiveShardCount.DEFAULT); // <2>
+            // end::create-index-request-waitForActiveShards
+
+            // tag::create-index-execute
+            CreateIndexResponse createIndexResponse = client.indices().createIndex(request);
+            // end::create-index-execute
+
+            // tag::create-index-response
+            boolean acknowledged = createIndexResponse.isAcknowledged(); // <1>
+            boolean shardsAcked = createIndexResponse.isShardsAcked(); // <2>
+            // end::create-index-response
+            assertTrue(acknowledged);
+            assertTrue(shardsAcked);
+
+            // tag::create-index-execute-async
+            client.indices().createIndexAsync(request, new ActionListener() {
+                @Override
+                public void onResponse(CreateIndexResponse createIndexResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            });
+            // end::create-index-execute-async
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java
index a9e4c777784..dc088f815b1 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/Alias.java
@@ -21,10 +21,13 @@ package org.elasticsearch.action.admin.indices.alias;
 
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -33,11 +36,17 @@ import org.elasticsearch.index.query.QueryBuilder;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * Represents an alias, to be associated with an index
  */
-public class Alias implements Streamable {
+public class Alias implements Streamable, ToXContentObject {
+
+    private static final ParseField FILTER = new ParseField("filter");
+    private static final ParseField ROUTING = new ParseField("routing");
+    private static final ParseField INDEX_ROUTING = new ParseField("index_routing", "indexRouting", "index-routing");
+    private static final ParseField SEARCH_ROUTING = new ParseField("search_routing", "searchRouting", "search-routing");
 
     private String name;
 
@@ -196,16 +205,16 @@ public class Alias implements Streamable {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if ("filter".equals(currentFieldName)) {
+                if (FILTER.match(currentFieldName)) {
                     Map filter = parser.mapOrdered();
                     alias.filter(filter);
                 }
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if ("routing".equals(currentFieldName)) {
+                if (ROUTING.match(currentFieldName)) {
                     alias.routing(parser.text());
-                } else if ("index_routing".equals(currentFieldName) || "indexRouting".equals(currentFieldName) || "index-routing".equals(currentFieldName)) {
+                } else if (INDEX_ROUTING.match(currentFieldName)) {
                     alias.indexRouting(parser.text());
-                } else if ("search_routing".equals(currentFieldName) || "searchRouting".equals(currentFieldName) || "search-routing".equals(currentFieldName)) {
+                } else if (SEARCH_ROUTING.match(currentFieldName)) {
                     alias.searchRouting(parser.text());
                 }
             }
@@ -213,6 +222,29 @@ public class Alias implements Streamable {
         return alias;
     }
 
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(name);
+
+        if (filter != null) {
+            builder.rawField(FILTER.getPreferredName(), new BytesArray(filter), XContentType.JSON);
+        }
+
+        if (indexRouting != null && indexRouting.equals(searchRouting)) {
+            builder.field(ROUTING.getPreferredName(), indexRouting);
+        } else {
+            if (indexRouting != null) {
+                builder.field(INDEX_ROUTING.getPreferredName(), indexRouting);
+            }
+            if (searchRouting != null) {
+                builder.field(SEARCH_ROUTING.getPreferredName(), searchRouting);
+            }
+        }
+
+        builder.endObject();
+        return builder;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
index 2d320b094b2..f628974834c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
@@ -30,6 +30,7 @@ import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.MapBuilder;
@@ -37,6 +38,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -65,7 +67,11 @@ import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
  * @see org.elasticsearch.client.Requests#createIndexRequest(String)
  * @see CreateIndexResponse
  */
-public class CreateIndexRequest extends AcknowledgedRequest implements IndicesRequest {
+public class CreateIndexRequest extends AcknowledgedRequest implements IndicesRequest, ToXContentObject {
+
+    private static final ParseField MAPPINGS = new ParseField("mappings");
+    private static final ParseField SETTINGS = new ParseField("settings");
+    private static final ParseField ALIASES = new ParseField("aliases");
 
     private String cause = "";
 
@@ -376,14 +382,14 @@ public class CreateIndexRequest extends AcknowledgedRequest
     public CreateIndexRequest source(Map source) {
         for (Map.Entry entry : source.entrySet()) {
             String name = entry.getKey();
-            if (name.equals("settings")) {
+            if (SETTINGS.match(name)) {
                 settings((Map) entry.getValue());
-            } else if (name.equals("mappings")) {
+            } else if (MAPPINGS.match(name)) {
                 Map mappings = (Map) entry.getValue();
                 for (Map.Entry entry1 : mappings.entrySet()) {
                     mapping(entry1.getKey(), (Map) entry1.getValue());
                 }
-            } else if (name.equals("aliases")) {
+            } else if (ALIASES.match(name)) {
                 aliases((Map) entry.getValue());
             } else {
                 // maybe custom?
@@ -520,4 +526,32 @@ public class CreateIndexRequest extends AcknowledgedRequest
         out.writeBoolean(updateAllTypes);
         waitForActiveShards.writeTo(out);
     }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder.startObject(SETTINGS.getPreferredName());
+        settings.toXContent(builder, params);
+        builder.endObject();
+
+        builder.startObject(MAPPINGS.getPreferredName());
+        for (Map.Entry entry : mappings.entrySet()) {
+            builder.rawField(entry.getKey(), new BytesArray(entry.getValue()), XContentType.JSON);
+        }
+        builder.endObject();
+
+        builder.startObject(ALIASES.getPreferredName());
+        for (Alias alias : aliases) {
+            alias.toXContent(builder, params);
+        }
+        builder.endObject();
+
+        for (Map.Entry entry : customs.entrySet()) {
+            builder.field(entry.getKey(), entry.getValue(), params);
+        }
+
+        builder.endObject();
+        return builder;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
index b770c11c6ab..5c07b4024ee 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
@@ -39,20 +39,17 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constru
  */
 public class CreateIndexResponse extends AcknowledgedResponse implements ToXContentObject {
 
-    private static final String SHARDS_ACKNOWLEDGED = "shards_acknowledged";
-    private static final String INDEX = "index";
-
-    private static final ParseField SHARDS_ACKNOWLEDGED_PARSER = new ParseField(SHARDS_ACKNOWLEDGED);
-    private static final ParseField INDEX_PARSER = new ParseField(INDEX);
+    private static final ParseField SHARDS_ACKNOWLEDGED = new ParseField("shards_acknowledged");
+    private static final ParseField INDEX = new ParseField("index");
 
     private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("create_index",
         true, args -> new CreateIndexResponse((boolean) args[0], (boolean) args[1], (String) args[2]));
 
     static {
         declareAcknowledgedField(PARSER);
-        PARSER.declareField(constructorArg(), (parser, context) -> parser.booleanValue(), SHARDS_ACKNOWLEDGED_PARSER,
+        PARSER.declareField(constructorArg(), (parser, context) -> parser.booleanValue(), SHARDS_ACKNOWLEDGED,
             ObjectParser.ValueType.BOOLEAN);
-        PARSER.declareField(constructorArg(), (parser, context) -> parser.text(), INDEX_PARSER, ObjectParser.ValueType.STRING);
+        PARSER.declareField(constructorArg(), (parser, context) -> parser.text(), INDEX, ObjectParser.ValueType.STRING);
     }
 
     private boolean shardsAcked;
@@ -102,8 +99,8 @@ public class CreateIndexResponse extends AcknowledgedResponse implements ToXCont
     }
 
     public void addCustomFields(XContentBuilder builder) throws IOException {
-        builder.field(SHARDS_ACKNOWLEDGED, isShardsAcked());
-        builder.field(INDEX, index());
+        builder.field(SHARDS_ACKNOWLEDGED.getPreferredName(), isShardsAcked());
+        builder.field(INDEX.getPreferredName(), index());
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
index e4467964722..3cce3d554f0 100755
--- a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
@@ -37,11 +37,10 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constru
  */
 public abstract class AcknowledgedResponse extends ActionResponse {
 
-    private static final String ACKNOWLEDGED = "acknowledged";
-    private static final ParseField ACKNOWLEDGED_PARSER = new ParseField(ACKNOWLEDGED);
+    private static final ParseField ACKNOWLEDGED = new ParseField("acknowledged");
 
     protected static  void declareAcknowledgedField(ConstructingObjectParser PARSER) {
-        PARSER.declareField(constructorArg(), (parser, context) -> parser.booleanValue(), ACKNOWLEDGED_PARSER,
+        PARSER.declareField(constructorArg(), (parser, context) -> parser.booleanValue(), ACKNOWLEDGED,
             ObjectParser.ValueType.BOOLEAN);
     }
 
@@ -78,6 +77,6 @@ public abstract class AcknowledgedResponse extends ActionResponse {
     }
 
     protected void addAcknowledgedField(XContentBuilder builder) throws IOException {
-        builder.field(ACKNOWLEDGED, isAcknowledged());
+        builder.field(ACKNOWLEDGED.getPreferredName(), isAcknowledged());
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
index 3648abb78c0..0a0a01c3fe3 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -624,7 +624,7 @@ public final class Settings implements ToXContentFragment {
     }
 
     /**
-     * Parsers the generated xconten from {@link Settings#toXContent(XContentBuilder, Params)} into a new Settings object.
+     * Parsers the generated xcontent from {@link Settings#toXContent(XContentBuilder, Params)} into a new Settings object.
      * Note this method requires the parser to either be positioned on a null token or on
      * {@link org.elasticsearch.common.xcontent.XContentParser.Token#START_OBJECT}.
      */
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java
index 4acdfd636bf..41691f70c06 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java
@@ -20,13 +20,26 @@
 package org.elasticsearch.action.admin.indices.create;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
 
 public class CreateIndexRequestTests extends ESTestCase {
 
@@ -46,7 +59,7 @@ public class CreateIndexRequestTests extends ESTestCase {
             }
         }
     }
-    
+
     public void testTopLevelKeys() throws IOException {
         String createIndex =
                 "{\n"
@@ -65,8 +78,168 @@ public class CreateIndexRequestTests extends ESTestCase {
                 + "}";
 
         CreateIndexRequest request = new CreateIndexRequest();
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, 
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class,
                 () -> {request.source(createIndex, XContentType.JSON);});
         assertEquals("unknown key [FOO_SHOULD_BE_ILLEGAL_HERE] for create index", e.getMessage());
     }
+
+    public void testToXContent() throws IOException {
+        CreateIndexRequest request = new CreateIndexRequest("foo");
+
+        String mapping = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().string();
+        request.mapping("my_type", mapping, XContentType.JSON);
+
+        Alias alias = new Alias("test_alias");
+        alias.routing("1");
+        alias.filter("{\"term\":{\"year\":2016}}");
+        request.alias(alias);
+
+        Settings.Builder settings = Settings.builder();
+        settings.put(SETTING_NUMBER_OF_SHARDS, 10);
+        request.settings(settings);
+
+        String actualRequestBody = Strings.toString(request);
+
+        String expectedRequestBody = "{\"settings\":{\"index\":{\"number_of_shards\":\"10\"}}," +
+            "\"mappings\":{\"my_type\":{\"type\":{}}}," +
+            "\"aliases\":{\"test_alias\":{\"filter\":{\"term\":{\"year\":2016}},\"routing\":\"1\"}}}";
+
+        assertEquals(expectedRequestBody, actualRequestBody);
+    }
+
+    public void testToAndFromXContent() throws IOException {
+
+        final CreateIndexRequest createIndexRequest = createTestItem();
+
+        boolean humanReadable = randomBoolean();
+        final XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference originalBytes = toShuffledXContent(createIndexRequest, xContentType, EMPTY_PARAMS, humanReadable);
+
+        CreateIndexRequest parsedCreateIndexRequest = new CreateIndexRequest(createIndexRequest.index());
+        parsedCreateIndexRequest.source(originalBytes, xContentType);
+
+        assertMappingsEqual(createIndexRequest.mappings(), parsedCreateIndexRequest.mappings());
+        assertAliasesEqual(createIndexRequest.aliases(), parsedCreateIndexRequest.aliases());
+        assertEquals(createIndexRequest.settings(), parsedCreateIndexRequest.settings());
+    }
+
+    private void assertMappingsEqual(Map expected, Map actual) throws IOException {
+        assertEquals(expected.keySet(), actual.keySet());
+
+        for (Map.Entry expectedEntry : expected.entrySet()) {
+            String expectedValue = expectedEntry.getValue();
+            String actualValue = actual.get(expectedEntry.getKey());
+            XContentParser expectedJson = createParser(XContentType.JSON.xContent(), expectedValue);
+            XContentParser actualJson = createParser(XContentType.JSON.xContent(), actualValue);
+            assertEquals(expectedJson.mapOrdered(), actualJson.mapOrdered());
+        }
+    }
+
+    private static void assertAliasesEqual(Set expected, Set actual) throws IOException {
+        assertEquals(expected, actual);
+
+        for (Alias expectedAlias : expected) {
+            for (Alias actualAlias : actual) {
+                if (expectedAlias.equals(actualAlias)) {
+                    // As Alias#equals only looks at name, we check the equality of the other Alias parameters here.
+                    assertEquals(expectedAlias.filter(), actualAlias.filter());
+                    assertEquals(expectedAlias.indexRouting(), actualAlias.indexRouting());
+                    assertEquals(expectedAlias.searchRouting(), actualAlias.searchRouting());
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns a random {@link CreateIndexRequest}.
+     */
+    private static CreateIndexRequest createTestItem() throws IOException {
+        String index = randomAlphaOfLength(5);
+
+        CreateIndexRequest request = new CreateIndexRequest(index);
+
+        int aliasesNo = randomIntBetween(0, 2);
+        for (int i = 0; i < aliasesNo; i++) {
+            request.alias(randomAlias());
+        }
+
+        if (randomBoolean()) {
+            String type = randomAlphaOfLength(5);
+            request.mapping(type, randomMapping(type));
+        }
+
+        if (randomBoolean()) {
+            request.settings(randomIndexSettings());
+        }
+
+        return request;
+    }
+
+    private static Settings randomIndexSettings() {
+        Settings.Builder builder = Settings.builder();
+
+        if (randomBoolean()) {
+            int numberOfShards = randomIntBetween(1, 10);
+            builder.put(SETTING_NUMBER_OF_SHARDS, numberOfShards);
+        }
+
+        if (randomBoolean()) {
+            int numberOfReplicas = randomIntBetween(1, 10);
+            builder.put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas);
+        }
+
+        return builder.build();
+    }
+
+    private static XContentBuilder randomMapping(String type) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject().startObject(type);
+
+        randomMappingFields(builder, true);
+
+        builder.endObject().endObject();
+        return builder;
+    }
+
+    private static void randomMappingFields(XContentBuilder builder, boolean allowObjectField) throws IOException {
+        builder.startObject("properties");
+
+        int fieldsNo = randomIntBetween(0, 5);
+        for (int i = 0; i < fieldsNo; i++) {
+            builder.startObject(randomAlphaOfLength(5));
+
+            if (allowObjectField && randomBoolean()) {
+                randomMappingFields(builder, false);
+            } else {
+                builder.field("type", "text");
+            }
+
+            builder.endObject();
+        }
+
+        builder.endObject();
+    }
+
+    private static Alias randomAlias() {
+        Alias alias = new Alias(randomAlphaOfLength(5));
+
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                alias.routing(randomAlphaOfLength(5));
+            } else {
+                if (randomBoolean()) {
+                    alias.indexRouting(randomAlphaOfLength(5));
+                }
+                if (randomBoolean()) {
+                    alias.searchRouting(randomAlphaOfLength(5));
+                }
+            }
+        }
+
+        if (randomBoolean()) {
+            alias.filter("{\"term\":{\"year\":2016}}");
+        }
+
+        return alias;
+    }
 }
diff --git a/docs/java-rest/high-level/apis/createindex.asciidoc b/docs/java-rest/high-level/apis/createindex.asciidoc
new file mode 100644
index 00000000000..ebd9158e193
--- /dev/null
+++ b/docs/java-rest/high-level/apis/createindex.asciidoc
@@ -0,0 +1,97 @@
+[[java-rest-high-create-index]]
+=== Create Index API
+
+[[java-rest-high-create-index-request]]
+==== Create Index Request
+
+A `CreateIndexRequest` requires an `index` argument:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[create-index-request]
+--------------------------------------------------
+<1> The index to create
+
+==== Index settings
+Each index created can have specific settings associated with it.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[create-index-request-settings]
+--------------------------------------------------
+<1> Settings for this index
+
+==== Index mappings
+An index may be created with mappings for its document types
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[create-index-request-mappings]
+--------------------------------------------------
+<1> The type to define
+<2> The mapping for this type, provided as a JSON string
+
+==== Index aliases
+Aliases can be set at index creation time
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[create-index-request-aliases]
+--------------------------------------------------
+<1> The alias to define
+
+==== Optional arguments
+The following arguments can optionally be provided:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[create-index-request-timeout]
+--------------------------------------------------
+<1> Timeout to wait for the all the nodes to acknowledge the index creation as a `TimeValue`
+<2> Timeout to wait for the all the nodes to acknowledge the index creatiom as a `String`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[create-index-request-masterTimeout]
+--------------------------------------------------
+<1> Timeout to connect to the master node as a `TimeValue`
+<2> Timeout to connect to the master node as a `String`
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[create-index-request-waitForActiveShards]
+--------------------------------------------------
+<1> The number of active shard copies to wait for before proceeding with the operation, as an `int`.
+<2> The number of active shard copies to wait for before proceeding with the operation, as an `ActiveShardCount`.
+
+[[java-rest-high-create-index-sync]]
+==== Synchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[create-index-execute]
+--------------------------------------------------
+
+[[java-rest-high-create-index-async]]
+==== Asynchronous Execution
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[create-index-execute-async]
+--------------------------------------------------
+<1> Called when the execution is successfully completed. The response is
+provided as an argument
+<2> Called in case of failure. The raised exception is provided as an argument
+
+[[java-rest-high-create-index-response]]
+==== Create Index Response
+
+The returned `CreateIndexResponse` allows to retrieve information about the executed
+ operation as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[create-index-response]
+--------------------------------------------------
+<1> Indicates whether all of the nodes have acknowledged the request
+<2> Indicates whether the requisite number of shard copies were started for each shard in the index before timing out
diff --git a/docs/java-rest/high-level/apis/deleteindex.asciidoc b/docs/java-rest/high-level/apis/deleteindex.asciidoc
index 3c0627de49a..e256790cf96 100644
--- a/docs/java-rest/high-level/apis/deleteindex.asciidoc
+++ b/docs/java-rest/high-level/apis/deleteindex.asciidoc
@@ -65,7 +65,7 @@ The returned `DeleteIndexResponse` allows to retrieve information about the exec
 --------------------------------------------------
 include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[delete-index-response]
 --------------------------------------------------
-<1> Indicates whether all of the nodes have acknowledged the request or not
+<1> Indicates whether all of the nodes have acknowledged the request
 
 If the index was not found, an `ElasticsearchException` will be thrown:
 
diff --git a/docs/java-rest/high-level/apis/index.asciidoc b/docs/java-rest/high-level/apis/index.asciidoc
index 993951b5ae7..2312f283720 100644
--- a/docs/java-rest/high-level/apis/index.asciidoc
+++ b/docs/java-rest/high-level/apis/index.asciidoc
@@ -1,3 +1,4 @@
+include::createindex.asciidoc[]
 include::deleteindex.asciidoc[]
 include::_index.asciidoc[]
 include::get.asciidoc[]
diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc
index 9e902e17157..7a6b55619f7 100644
--- a/docs/java-rest/high-level/supported-apis.asciidoc
+++ b/docs/java-rest/high-level/supported-apis.asciidoc
@@ -4,6 +4,7 @@
 The Java High Level REST Client supports the following APIs:
 
 Indices APIs::
+* <>
 * <>
 
 Single document APIs::

From 69dd667f5e4f5251e284c37a0b6e7483cc45f082 Mon Sep 17 00:00:00 2001
From: Yannick Welsch 
Date: Thu, 7 Dec 2017 11:51:49 +0100
Subject: [PATCH 203/297] Add unreleased v6.0.2 version

---
 core/src/main/java/org/elasticsearch/Version.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index a4507fc91e2..e234e8828bc 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -131,6 +131,9 @@ public class Version implements Comparable {
     public static final int V_6_0_1_ID = 6000199;
     public static final Version V_6_0_1 =
         new Version(V_6_0_1_ID, org.apache.lucene.util.Version.LUCENE_7_0_1);
+    public static final int V_6_0_2_ID = 6000299;
+    public static final Version V_6_0_2 =
+        new Version(V_6_0_2_ID, org.apache.lucene.util.Version.LUCENE_7_0_1);
     public static final int V_6_1_0_ID = 6010099;
     public static final Version V_6_1_0 = new Version(V_6_1_0_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
     public static final int V_6_2_0_ID = 6020099;
@@ -157,6 +160,8 @@ public class Version implements Comparable {
                 return V_6_1_0;
             case V_6_2_0_ID:
                 return V_6_2_0;
+            case V_6_0_2_ID:
+                return V_6_0_2;
             case V_6_0_1_ID:
                 return V_6_0_1;
             case V_6_0_0_ID:

From ee21045697800810b07140729ae9289400d5d74f Mon Sep 17 00:00:00 2001
From: markwalkom 
Date: Fri, 8 Dec 2017 00:06:58 +1100
Subject: [PATCH 204/297] [Docs] Add date math examples to
 api-conventions.asciidoc (#25217)

---
 docs/reference/api-conventions.asciidoc | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc
index 3d8cba18e05..db138912683 100644
--- a/docs/reference/api-conventions.asciidoc
+++ b/docs/reference/api-conventions.asciidoc
@@ -215,13 +215,12 @@ The supported units are:
 `m`:: minutes
 `s`:: seconds
 
-Some examples are:
+Assuming `now` is `2001-01-01 12:00:00`, some examples are:
 
-[horizontal]
-`now+1h`::              The current time plus one hour, with ms resolution.
-`now+1h+1m`::           The current time plus one hour plus one minute, with ms resolution.
-`now+1h/d`::            The current time plus one hour, rounded down to the nearest day.
-`2015-01-01||+1M/d`::   `2015-01-01` plus one month, rounded down to the nearest day.
+`now+1h`:: `now` in milliseconds plus one hour. Resolves to: `2001-01-01 13:00:00`
+`now-1h`:: `now` in milliseconds plus one hour. Resolves to: `2001-01-01 11:00:00`
+`now-1h/d`:: `now` in milliseconds rounded down to UTC 00:00. Resolves to: `2001-01-01 00:00:00``
+ `2001-01-01\|\|+1M/d`:: `now` in milliseconds plus one month. Resolves to: `2001-02-01 00:00:00`
 
 [float]
 [[common-options-response-filtering]]

From 057efea8933adb2f491dd52ba252441063facce2 Mon Sep 17 00:00:00 2001
From: Robin Neatherway 
Date: Thu, 7 Dec 2017 13:18:11 +0000
Subject: [PATCH 205/297] Correct two equality checks on incomparable types
 (#27688)

---
 .../org/elasticsearch/index/mapper/GeoShapeFieldMapper.java     | 2 +-
 .../src/main/java/org/elasticsearch/painless/Definition.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
index db40fb228bc..68d6ac66678 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
@@ -369,7 +369,7 @@ public class GeoShapeFieldMapper extends FieldMapper {
         public void setStrategyName(String strategyName) {
             checkIfFrozen();
             this.strategyName = strategyName;
-            if (this.strategyName.equals(SpatialStrategy.TERM)) {
+            if (this.strategyName.equals(SpatialStrategy.TERM.getStrategyName())) {
                 this.pointsOnly = true;
             }
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index e5bfb82c731..df56c599f03 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -768,7 +768,7 @@ public final class Definition {
             painlessConstructor = new Method("", ownerStruct, null, getTypeInternal("void"), painlessParametersTypes,
                 asmConstructor, javaConstructor.getModifiers(), javaHandle);
             ownerStruct.constructors.put(painlessMethodKey, painlessConstructor);
-        } else if (painlessConstructor.equals(painlessParametersTypes) == false){
+        } else if (painlessConstructor.arguments.equals(painlessParametersTypes) == false){
             throw new IllegalArgumentException(
                     "illegal duplicate constructors [" + painlessMethodKey + "] found within the struct [" + ownerStruct.name + "] " +
                     "with parameters " + painlessParametersTypes + " and " + painlessConstructor.arguments);

From 5a53798f83d0d3b6969e9925fbeec7571bc1f95e Mon Sep 17 00:00:00 2001
From: Yannick Welsch 
Date: Thu, 7 Dec 2017 14:58:59 +0100
Subject: [PATCH 206/297] Add unreleased v5.6.6 version

---
 core/src/main/java/org/elasticsearch/Version.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index e234e8828bc..cfdff404d32 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -107,6 +107,8 @@ public class Version implements Comparable {
     public static final Version V_5_6_4 = new Version(V_5_6_4_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
     public static final int V_5_6_5_ID = 5060599;
     public static final Version V_5_6_5 = new Version(V_5_6_5_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
+    public static final int V_5_6_6_ID = 5060699;
+    public static final Version V_5_6_6 = new Version(V_5_6_6_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
     public static final int V_6_0_0_alpha1_ID = 6000001;
     public static final Version V_6_0_0_alpha1 =
             new Version(V_6_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0);
@@ -178,6 +180,8 @@ public class Version implements Comparable {
                 return V_6_0_0_alpha2;
             case V_6_0_0_alpha1_ID:
                 return V_6_0_0_alpha1;
+            case V_5_6_6_ID:
+                return V_5_6_6;
             case V_5_6_5_ID:
                 return V_5_6_5;
             case V_5_6_4_ID:

From b83e14858a920a55681249eb0b3e3249724c39c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?=
 <10398885+cbuescher@users.noreply.github.com>
Date: Thu, 7 Dec 2017 16:39:23 +0100
Subject: [PATCH 207/297] Correcting some minor typos in comments

---
 .../action/search/TransportMultiSearchAction.java            | 4 ++--
 .../main/java/org/elasticsearch/indices/IndicesService.java  | 2 +-
 .../java/org/elasticsearch/percolator/QueryAnalyzer.java     | 2 +-
 .../org/elasticsearch/index/shard/IndexShardTestCase.java    | 5 ++---
 .../org/elasticsearch/test/disruption/LongGCDisruption.java  | 2 +-
 5 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
index 371314b990c..16d1deee1f9 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
@@ -71,7 +71,7 @@ public class TransportMultiSearchAction extends HandledTransportAction listener) {
         final long relativeStartTime = relativeTimeProvider.getAsLong();
-        
+
         ClusterState clusterState = clusterService.state();
         clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
 
@@ -130,7 +130,7 @@ public class TransportMultiSearchAction extends HandledTransportAction suspendingError = new AtomicReference<>();
                 final Thread suspendingThread = new Thread(new AbstractRunnable() {

From bcc33f391fd0790fc8c37488be32a986e4d9287e Mon Sep 17 00:00:00 2001
From: olcbean <26058559+olcbean@users.noreply.github.com>
Date: Thu, 7 Dec 2017 18:16:03 +0100
Subject: [PATCH 208/297] Add Open Index API to the high level REST client
 (#27574)

Add _open to the high level REST client

Relates to #27205
---
 .../elasticsearch/client/IndicesClient.java   | 27 ++++++-
 .../org/elasticsearch/client/Request.java     | 14 ++++
 .../client/RestHighLevelClient.java           |  4 +-
 .../elasticsearch/client/IndicesClientIT.java | 74 +++++++++++++++++--
 .../elasticsearch/client/RequestTests.java    | 36 +++++++--
 .../admin/indices/open/OpenIndexResponse.java | 34 ++++++++-
 .../indices/open/OpenIndexResponseTests.java  | 63 ++++++++++++++++
 7 files changed, 238 insertions(+), 14 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
index c8767318399..57dafbba509 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
@@ -25,6 +25,8 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -72,7 +74,7 @@ public final class IndicesClient {
      */
     public CreateIndexResponse createIndex(CreateIndexRequest createIndexRequest, Header... headers) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(createIndexRequest, Request::createIndex, CreateIndexResponse::fromXContent,
-            Collections.emptySet(), headers);
+                Collections.emptySet(), headers);
     }
 
     /**
@@ -86,4 +88,27 @@ public final class IndicesClient {
         restHighLevelClient.performRequestAsyncAndParseEntity(createIndexRequest, Request::createIndex, CreateIndexResponse::fromXContent,
             listener, Collections.emptySet(), headers);
     }
+
+    /**
+     * Opens an index using the Open Index API
+     * 
    
+     * See 
+     * Open Index API on elastic.co
+     */
+    public OpenIndexResponse openIndex(OpenIndexRequest openIndexRequest, Header... headers) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(openIndexRequest, Request::openIndex, OpenIndexResponse::fromXContent,
+                Collections.emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously opens an index using the Open Index API
+     * 
    
+     * See 
+     * Open Index API on elastic.co
+     */
+    public void openIndexAsync(OpenIndexRequest openIndexRequest, ActionListener listener, Header... headers) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(openIndexRequest, Request::openIndex, OpenIndexResponse::fromXContent,
+                listener, Collections.emptySet(), headers);
+    }
+
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
index a3544ddb89b..dd08179cf62 100755
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
@@ -31,6 +31,7 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.get.GetRequest;
@@ -138,6 +139,19 @@ public final class Request {
         return new Request(HttpDelete.METHOD_NAME, endpoint, parameters.getParams(), null);
     }
 
+    static Request openIndex(OpenIndexRequest openIndexRequest) {
+        String endpoint = endpoint(openIndexRequest.indices(), Strings.EMPTY_ARRAY, "_open");
+
+        Params parameters = Params.builder();
+
+        parameters.withTimeout(openIndexRequest.timeout());
+        parameters.withMasterTimeout(openIndexRequest.masterNodeTimeout());
+        parameters.withWaitForActiveShards(openIndexRequest.waitForActiveShards());
+        parameters.withIndicesOptions(openIndexRequest.indicesOptions());
+
+        return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), null);
+    }
+
     static Request createIndex(CreateIndexRequest createIndexRequest) throws IOException {
         String endpoint = endpoint(createIndexRequest.indices(), Strings.EMPTY_ARRAY, "");
 
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
index 29ab7f90ff5..ca244eee88c 100755
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -26,6 +26,8 @@ import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -283,7 +285,7 @@ public class RestHighLevelClient implements Closeable {
      *
      * See Get API on elastic.co
      */
-    public void getAsync(GetRequest getRequest, ActionListener listener, Header... headers) {
+    public final void getAsync(GetRequest getRequest, ActionListener listener, Header... headers) {
         performRequestAsyncAndParseEntity(getRequest, Request::get, GetResponse::fromXContent, listener, singleton(404), headers);
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
index 0d6430b5912..5f356c4c29f 100755
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
@@ -25,14 +25,21 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import static org.hamcrest.Matchers.equalTo;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.rest.RestStatus;
 
-import java.io.IOException;
 import java.util.Map;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@@ -128,16 +135,73 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase {
         }
     }
 
+    public void testOpenExistingIndex() throws IOException {
+        String[] indices = randomIndices(1, 5);
+        for (String index : indices) {
+            createIndex(index);
+            closeIndex(index);
+            ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest("GET", index + "/_search"));
+            assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus()));
+            assertThat(exception.getMessage().contains(index), equalTo(true));
+        }
+
+        OpenIndexRequest openIndexRequest = new OpenIndexRequest(indices);
+        OpenIndexResponse openIndexResponse = execute(openIndexRequest, highLevelClient().indices()::openIndex,
+                highLevelClient().indices()::openIndexAsync);
+        assertTrue(openIndexResponse.isAcknowledged());
+
+        for (String index : indices) {
+            Response response = client().performRequest("GET", index + "/_search");
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+        }
+    }
+
+    public void testOpenNonExistentIndex() throws IOException {
+        String[] nonExistentIndices = randomIndices(1, 5);
+        for (String nonExistentIndex : nonExistentIndices) {
+            assertFalse(indexExists(nonExistentIndex));
+        }
+
+        OpenIndexRequest openIndexRequest = new OpenIndexRequest(nonExistentIndices);
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                () -> execute(openIndexRequest, highLevelClient().indices()::openIndex, highLevelClient().indices()::openIndexAsync));
+        assertEquals(RestStatus.NOT_FOUND, exception.status());
+
+        OpenIndexRequest lenientOpenIndexRequest = new OpenIndexRequest(nonExistentIndices);
+        lenientOpenIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
+        OpenIndexResponse lenientOpenIndexResponse = execute(lenientOpenIndexRequest, highLevelClient().indices()::openIndex,
+                highLevelClient().indices()::openIndexAsync);
+        assertThat(lenientOpenIndexResponse.isAcknowledged(), equalTo(true));
+
+        OpenIndexRequest strictOpenIndexRequest = new OpenIndexRequest(nonExistentIndices);
+        strictOpenIndexRequest.indicesOptions(IndicesOptions.strictExpandOpen());
+        ElasticsearchException strictException = expectThrows(ElasticsearchException.class,
+                () -> execute(openIndexRequest, highLevelClient().indices()::openIndex, highLevelClient().indices()::openIndexAsync));
+        assertEquals(RestStatus.NOT_FOUND, strictException.status());
+    }
+
+    private static String[] randomIndices(int minIndicesNum, int maxIndicesNum) {
+        int numIndices = randomIntBetween(minIndicesNum, maxIndicesNum);
+        String[] indices = new String[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            indices[i] = "index-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT);
+        }
+        return indices;
+    }
+
     private static void createIndex(String index) throws IOException {
         Response response = client().performRequest("PUT", index);
-
-        assertEquals(200, response.getStatusLine().getStatusCode());
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
     }
 
     private static boolean indexExists(String index) throws IOException {
         Response response = client().performRequest("HEAD", index);
+        return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode();
+    }
 
-        return response.getStatusLine().getStatusCode() == 200;
+    private static void closeIndex(String index) throws IOException {
+        Response response = client().performRequest("POST", index + "/_close");
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
     }
 
     @SuppressWarnings("unchecked")
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
index 182de30fd15..f39a7c77b80 100755
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
@@ -27,6 +27,7 @@ import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -37,7 +38,6 @@ import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -83,7 +83,6 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.StringJoiner;
-import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -93,10 +92,12 @@ import static org.elasticsearch.client.Request.REQUEST_BODY_CONTENT_TYPE;
 import static org.elasticsearch.client.Request.enforceSameContentType;
 import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 public class RequestTests extends ESTestCase {
 
-    public void testConstructor() throws Exception {
+    public void testConstructor() {
         final String method = randomFrom("GET", "PUT", "POST", "HEAD", "DELETE");
         final String endpoint = randomAlphaOfLengthBetween(1, 10);
         final Map parameters = singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(5));
@@ -122,7 +123,7 @@ public class RequestTests extends ESTestCase {
         assertTrue("Request constructor is not public", Modifier.isPublic(constructors[0].getModifiers()));
     }
 
-    public void testClassVisibility() throws Exception {
+    public void testClassVisibility() {
         assertTrue("Request class is not public", Modifier.isPublic(Request.class.getModifiers()));
     }
 
@@ -146,7 +147,7 @@ public class RequestTests extends ESTestCase {
         getAndExistsTest(Request::get, "GET");
     }
 
-    public void testDelete() throws IOException {
+    public void testDelete() {
         String index = randomAlphaOfLengthBetween(3, 10);
         String type = randomAlphaOfLengthBetween(3, 10);
         String id = randomAlphaOfLengthBetween(3, 10);
@@ -283,7 +284,7 @@ public class RequestTests extends ESTestCase {
         assertToXContentBody(createIndexRequest, request.getEntity());
     }
 
-    public void testDeleteIndex() throws IOException {
+    public void testDeleteIndex() {
         DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest();
 
         int numIndices = randomIntBetween(0, 5);
@@ -307,6 +308,29 @@ public class RequestTests extends ESTestCase {
         assertNull(request.getEntity());
     }
 
+    public void testOpenIndex() {
+        OpenIndexRequest openIndexRequest = new OpenIndexRequest();
+        int numIndices = randomIntBetween(1, 5);
+        String[] indices = new String[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            indices[i] = "index-" + randomAlphaOfLengthBetween(2, 5);
+        }
+        openIndexRequest.indices(indices);
+
+        Map expectedParams = new HashMap<>();
+        setRandomTimeout(openIndexRequest::timeout, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams);
+        setRandomMasterTimeout(openIndexRequest, expectedParams);
+        setRandomIndicesOptions(openIndexRequest::indicesOptions, openIndexRequest::indicesOptions, expectedParams);
+        setRandomWaitForActiveShards(openIndexRequest::waitForActiveShards, expectedParams);
+
+        Request request = Request.openIndex(openIndexRequest);
+        StringJoiner endpoint = new StringJoiner("/", "/", "").add(String.join(",", indices)).add("_open");
+        assertThat(endpoint.toString(), equalTo(request.getEndpoint()));
+        assertThat(expectedParams, equalTo(request.getParameters()));
+        assertThat(request.getMethod(), equalTo("POST"));
+        assertThat(request.getEntity(), nullValue());
+    }
+
     public void testIndex() throws IOException {
         String index = randomAlphaOfLengthBetween(3, 10);
         String type = randomAlphaOfLengthBetween(3, 10);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java
index fe9343f363f..95fef9fc653 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java
@@ -21,15 +21,34 @@ package org.elasticsearch.action.admin.indices.open;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
 /**
  * A response for a open index action.
  */
-public class OpenIndexResponse extends AcknowledgedResponse {
+public class OpenIndexResponse extends AcknowledgedResponse implements ToXContentObject  {
+    private static final String SHARDS_ACKNOWLEDGED = "shards_acknowledged";
+    private static final ParseField SHARDS_ACKNOWLEDGED_PARSER = new ParseField(SHARDS_ACKNOWLEDGED);
+
+    private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("open_index", true,
+            args -> new OpenIndexResponse((boolean) args[0], (boolean) args[1]));
+
+    static {
+        declareAcknowledgedField(PARSER);
+        PARSER.declareField(constructorArg(), (parser, context) -> parser.booleanValue(), SHARDS_ACKNOWLEDGED_PARSER,
+                ObjectParser.ValueType.BOOLEAN);
+    }
 
     private boolean shardsAcknowledged;
 
@@ -68,4 +87,17 @@ public class OpenIndexResponse extends AcknowledgedResponse {
             out.writeBoolean(shardsAcknowledged);
         }
     }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        addAcknowledgedField(builder);
+        builder.field(SHARDS_ACKNOWLEDGED, isShardsAcknowledged());
+        builder.endObject();
+        return builder;
+    }
+
+    public static OpenIndexResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.apply(parser, null);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java
new file mode 100644
index 00000000000..09ceb796034
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.open;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class OpenIndexResponseTests extends ESTestCase {
+
+    public void testFromToXContent() throws IOException {
+        final OpenIndexResponse openIndexResponse = createTestItem();
+
+        boolean humanReadable = randomBoolean();
+        final XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference originalBytes = toShuffledXContent(openIndexResponse, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        BytesReference mutated;
+        if (randomBoolean()) {
+            mutated = insertRandomFields(xContentType, originalBytes, null, random());
+        } else {
+            mutated = originalBytes;
+        }
+        
+        OpenIndexResponse parsedOpenIndexResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), mutated)) {
+            parsedOpenIndexResponse = OpenIndexResponse.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        assertThat(parsedOpenIndexResponse.isShardsAcknowledged(), equalTo(openIndexResponse.isShardsAcknowledged()));
+        assertThat(parsedOpenIndexResponse.isAcknowledged(), equalTo(openIndexResponse.isAcknowledged()));
+    }
+    
+    private static OpenIndexResponse createTestItem() {
+        boolean acknowledged = randomBoolean();
+        boolean shardsAcked = acknowledged && randomBoolean();
+        return new OpenIndexResponse(acknowledged, shardsAcked);
+    }
+}

From cca54b811d0250b56a1ac24697baab6b63429316 Mon Sep 17 00:00:00 2001
From: Lee Hinman 
Date: Thu, 7 Dec 2017 11:57:22 -0700
Subject: [PATCH 209/297] [TEST] Wait for merging to complete before testing
 breaker

It's possible that a merge may be ongoing when we check the breaker and segment
stats' memory usage, this causes the test to fail. Instead, we should wait for
merging to complete.

Resolves #27651
---
 .../java/org/elasticsearch/index/shard/IndexShardTests.java   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index dc4294f30f5..ebf39b7e089 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -2854,6 +2854,10 @@ public class IndexShardTests extends IndexShardTestCase {
             t.join();
         }
 
+        // We need to wait for all ongoing merges to complete. The reason is that during a merge the
+        // IndexWriter holds the core cache key open and causes the memory to be registered in the breaker
+        primary.forceMerge(new ForceMergeRequest());
+
         // Close remaining searchers
         IOUtils.close(searchers);
 

From da5f52a2fcb828141748004c5586e87da6088395 Mon Sep 17 00:00:00 2001
From: Tim Brooks 
Date: Thu, 7 Dec 2017 12:48:49 -0700
Subject: [PATCH 210/297] Add test for writer operation buffer accounting
 (#27707)

This is a follow up to #27695. This commit adds a test checking that
across multiple writes using multiple buffers, a write operation
properly keeps track of which buffers still need to be written.
---
 .../transport/nio/WriteOperationTests.java    | 51 +++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/WriteOperationTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/WriteOperationTests.java
index 0015d39a373..0085a9b204a 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/WriteOperationTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/WriteOperationTests.java
@@ -21,12 +21,15 @@ package org.elasticsearch.transport.nio;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.CompositeBytesReference;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.nio.channel.NioSocketChannel;
 import org.junit.Before;
+import org.mockito.ArgumentCaptor;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.List;
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
@@ -65,4 +68,52 @@ public class WriteOperationTests extends ESTestCase {
 
         assertFalse(writeOp.isFullyFlushed());
     }
+
+    public void testMultipleFlushesWithCompositeBuffer() throws IOException {
+        BytesArray bytesReference1 = new BytesArray(new byte[10]);
+        BytesArray bytesReference2 = new BytesArray(new byte[15]);
+        BytesArray bytesReference3 = new BytesArray(new byte[3]);
+        CompositeBytesReference bytesReference = new CompositeBytesReference(bytesReference1, bytesReference2, bytesReference3);
+        WriteOperation writeOp = new WriteOperation(channel, bytesReference, listener);
+
+        ArgumentCaptor buffersCaptor = ArgumentCaptor.forClass(ByteBuffer[].class);
+
+        when(channel.write(buffersCaptor.capture())).thenReturn(5)
+            .thenReturn(5)
+            .thenReturn(2)
+            .thenReturn(15)
+            .thenReturn(1);
+
+        writeOp.flush();
+        assertFalse(writeOp.isFullyFlushed());
+        writeOp.flush();
+        assertFalse(writeOp.isFullyFlushed());
+        writeOp.flush();
+        assertFalse(writeOp.isFullyFlushed());
+        writeOp.flush();
+        assertFalse(writeOp.isFullyFlushed());
+        writeOp.flush();
+        assertTrue(writeOp.isFullyFlushed());
+
+        List values = buffersCaptor.getAllValues();
+        ByteBuffer[] byteBuffers = values.get(0);
+        assertEquals(3, byteBuffers.length);
+        assertEquals(10, byteBuffers[0].remaining());
+
+        byteBuffers = values.get(1);
+        assertEquals(3, byteBuffers.length);
+        assertEquals(5, byteBuffers[0].remaining());
+
+        byteBuffers = values.get(2);
+        assertEquals(2, byteBuffers.length);
+        assertEquals(15, byteBuffers[0].remaining());
+
+        byteBuffers = values.get(3);
+        assertEquals(2, byteBuffers.length);
+        assertEquals(13, byteBuffers[0].remaining());
+
+        byteBuffers = values.get(4);
+        assertEquals(1, byteBuffers.length);
+        assertEquals(1, byteBuffers[0].remaining());
+    }
 }

From 6efee323e094d007d9d8e6ba06f2479b4809ec94 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen 
Date: Thu, 7 Dec 2017 21:42:11 -0500
Subject: [PATCH 211/297] Remove unused *Commit* classes (#27714)

These classes are not used anywhere.
---
 .../common/lucene/IndexCommitDelegate.java    |  98 ---------
 .../index/shard/CommitPoint.java              | 145 -------------
 .../index/shard/CommitPoints.java             | 201 ------------------
 .../index/shard/CommitPointsTests.java        |  69 ------
 4 files changed, 513 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/common/lucene/IndexCommitDelegate.java
 delete mode 100644 core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
 delete mode 100644 core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
 delete mode 100644 core/src/test/java/org/elasticsearch/index/shard/CommitPointsTests.java

diff --git a/core/src/main/java/org/elasticsearch/common/lucene/IndexCommitDelegate.java b/core/src/main/java/org/elasticsearch/common/lucene/IndexCommitDelegate.java
deleted file mode 100644
index 7831a7568cf..00000000000
--- a/core/src/main/java/org/elasticsearch/common/lucene/IndexCommitDelegate.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.lucene;
-
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.store.Directory;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-
-/**
- * A simple delegate that delegates all {@link IndexCommit} calls to a delegated
- * {@link IndexCommit}.
- *
- *
- */
-public abstract class IndexCommitDelegate extends IndexCommit {
-
-    protected final IndexCommit delegate;
-
-    /**
-     * Constructs a new {@link IndexCommit} that will delegate all calls
-     * to the provided delegate.
-     *
-     * @param delegate The delegate
-     */
-    public IndexCommitDelegate(IndexCommit delegate) {
-        this.delegate = delegate;
-    }
-
-    @Override
-    public String getSegmentsFileName() {
-        return delegate.getSegmentsFileName();
-    }
-
-    @Override
-    public Collection getFileNames() throws IOException {
-        return delegate.getFileNames();
-    }
-
-    @Override
-    public Directory getDirectory() {
-        return delegate.getDirectory();
-    }
-
-    @Override
-    public void delete() {
-        delegate.delete();
-    }
-
-    @Override
-    public boolean isDeleted() {
-        return delegate.isDeleted();
-    }
-
-    @Override
-    public int getSegmentCount() {
-        return delegate.getSegmentCount();
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        return delegate.equals(other);
-    }
-
-    @Override
-    public int hashCode() {
-        return delegate.hashCode();
-    }
-
-    @Override
-    public long getGeneration() {
-        return delegate.getGeneration();
-    }
-
-    @Override
-    public Map getUserData() throws IOException {
-        return delegate.getUserData();
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java b/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
deleted file mode 100644
index 03afc356f31..00000000000
--- a/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.shard;
-
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.index.store.StoreFileMetaData;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-public class CommitPoint {
-
-    public static final CommitPoint NULL = new CommitPoint(-1, "_null_", Type.GENERATED, Collections.emptyList(), Collections.emptyList());
-
-    public static class FileInfo {
-        private final String name;
-        private final String physicalName;
-        private final long length;
-        private final String checksum;
-
-        public FileInfo(String name, String physicalName, long length, String checksum) {
-            this.name = name;
-            this.physicalName = physicalName;
-            this.length = length;
-            this.checksum = checksum;
-        }
-
-        public String name() {
-            return name;
-        }
-
-        public String physicalName() {
-            return this.physicalName;
-        }
-
-        public long length() {
-            return length;
-        }
-
-        @Nullable
-        public String checksum() {
-            return checksum;
-        }
-    }
-
-    public enum Type {
-        GENERATED,
-        SAVED
-    }
-
-    private final long version;
-
-    private final String name;
-
-    private final Type type;
-
-    private final List indexFiles;
-
-    private final List translogFiles;
-
-    public CommitPoint(long version, String name, Type type, List indexFiles, List translogFiles) {
-        this.version = version;
-        this.name = name;
-        this.type = type;
-        this.indexFiles = Collections.unmodifiableList(new ArrayList<>(indexFiles));
-        this.translogFiles = Collections.unmodifiableList(new ArrayList<>(translogFiles));
-    }
-
-    public long version() {
-        return version;
-    }
-
-    public String name() {
-        return this.name;
-    }
-
-    public Type type() {
-        return this.type;
-    }
-
-    public List indexFiles() {
-        return this.indexFiles;
-    }
-
-    public List translogFiles() {
-        return this.translogFiles;
-    }
-
-    public boolean containPhysicalIndexFile(String physicalName) {
-        return findPhysicalIndexFile(physicalName) != null;
-    }
-
-    public CommitPoint.FileInfo findPhysicalIndexFile(String physicalName) {
-        for (FileInfo file : indexFiles) {
-            if (file.physicalName().equals(physicalName)) {
-                return file;
-            }
-        }
-        return null;
-    }
-
-    public CommitPoint.FileInfo findNameFile(String name) {
-        CommitPoint.FileInfo fileInfo = findNameIndexFile(name);
-        if (fileInfo != null) {
-            return fileInfo;
-        }
-        return findNameTranslogFile(name);
-    }
-
-    public CommitPoint.FileInfo findNameIndexFile(String name) {
-        for (FileInfo file : indexFiles) {
-            if (file.name().equals(name)) {
-                return file;
-            }
-        }
-        return null;
-    }
-
-    public CommitPoint.FileInfo findNameTranslogFile(String name) {
-        for (FileInfo file : translogFiles) {
-            if (file.name().equals(name)) {
-                return file;
-            }
-        }
-        return null;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java b/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
deleted file mode 100644
index 12a31076228..00000000000
--- a/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.shard;
-
-import org.apache.lucene.util.CollectionUtil;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-
-public class CommitPoints implements Iterable {
-
-    private final List commitPoints;
-
-    public CommitPoints(List commitPoints) {
-        CollectionUtil.introSort(commitPoints, new Comparator() {
-            @Override
-            public int compare(CommitPoint o1, CommitPoint o2) {
-                return (o2.version() < o1.version() ? -1 : (o2.version() == o1.version() ? 0 : 1));
-            }
-        });
-        this.commitPoints = Collections.unmodifiableList(new ArrayList<>(commitPoints));
-    }
-
-    public List commits() {
-        return this.commitPoints;
-    }
-
-    public boolean hasVersion(long version) {
-        for (CommitPoint commitPoint : commitPoints) {
-            if (commitPoint.version() == version) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    public CommitPoint.FileInfo findPhysicalIndexFile(String physicalName) {
-        for (CommitPoint commitPoint : commitPoints) {
-            CommitPoint.FileInfo fileInfo = commitPoint.findPhysicalIndexFile(physicalName);
-            if (fileInfo != null) {
-                return fileInfo;
-            }
-        }
-        return null;
-    }
-
-    public CommitPoint.FileInfo findNameFile(String name) {
-        for (CommitPoint commitPoint : commitPoints) {
-            CommitPoint.FileInfo fileInfo = commitPoint.findNameFile(name);
-            if (fileInfo != null) {
-                return fileInfo;
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Iterator iterator() {
-        return commitPoints.iterator();
-    }
-
-    public static byte[] toXContent(CommitPoint commitPoint) throws Exception {
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
-        builder.startObject();
-        builder.field("version", commitPoint.version());
-        builder.field("name", commitPoint.name());
-        builder.field("type", commitPoint.type().toString());
-
-        builder.startObject("index_files");
-        for (CommitPoint.FileInfo fileInfo : commitPoint.indexFiles()) {
-            builder.startObject(fileInfo.name());
-            builder.field("physical_name", fileInfo.physicalName());
-            builder.field("length", fileInfo.length());
-            if (fileInfo.checksum() != null) {
-                builder.field("checksum", fileInfo.checksum());
-            }
-            builder.endObject();
-        }
-        builder.endObject();
-
-        builder.startObject("translog_files");
-        for (CommitPoint.FileInfo fileInfo : commitPoint.translogFiles()) {
-            builder.startObject(fileInfo.name());
-            builder.field("physical_name", fileInfo.physicalName());
-            builder.field("length", fileInfo.length());
-            builder.endObject();
-        }
-        builder.endObject();
-
-        builder.endObject();
-        return BytesReference.toBytes(builder.bytes());
-    }
-
-    public static CommitPoint fromXContent(byte[] data) throws Exception {
-        // EMPTY is safe here because we never call namedObject
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY, data)) {
-            String currentFieldName = null;
-            XContentParser.Token token = parser.nextToken();
-            if (token == null) {
-                // no data...
-                throw new IOException("No commit point data");
-            }
-            long version = -1;
-            String name = null;
-            CommitPoint.Type type = null;
-            List indexFiles = new ArrayList<>();
-            List translogFiles = new ArrayList<>();
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                } else if (token == XContentParser.Token.START_OBJECT) {
-                    List files = null;
-                    if ("index_files".equals(currentFieldName) || "indexFiles".equals(currentFieldName)) {
-                        files = indexFiles;
-                    } else if ("translog_files".equals(currentFieldName) || "translogFiles".equals(currentFieldName)) {
-                        files = translogFiles;
-                    } else {
-                        throw new IOException("Can't handle object with name [" + currentFieldName + "]");
-                    }
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            currentFieldName = parser.currentName();
-                        } else if (token == XContentParser.Token.START_OBJECT) {
-                            String fileName = currentFieldName;
-                            String physicalName = null;
-                            long size = -1;
-                            String checksum = null;
-                            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                                if (token == XContentParser.Token.FIELD_NAME) {
-                                    currentFieldName = parser.currentName();
-                                } else if (token.isValue()) {
-                                    if ("physical_name".equals(currentFieldName) || "physicalName".equals(currentFieldName)) {
-                                        physicalName = parser.text();
-                                    } else if ("length".equals(currentFieldName)) {
-                                        size = parser.longValue();
-                                    } else if ("checksum".equals(currentFieldName)) {
-                                        checksum = parser.text();
-                                    }
-                                }
-                            }
-                            if (physicalName == null) {
-                                throw new IOException("Malformed commit, missing physical_name for [" + fileName + "]");
-                            }
-                            if (size == -1) {
-                                throw new IOException("Malformed commit, missing length for [" + fileName + "]");
-                            }
-                            files.add(new CommitPoint.FileInfo(fileName, physicalName, size, checksum));
-                        }
-                    }
-                } else if (token.isValue()) {
-                    if ("version".equals(currentFieldName)) {
-                        version = parser.longValue();
-                    } else if ("name".equals(currentFieldName)) {
-                        name = parser.text();
-                    } else if ("type".equals(currentFieldName)) {
-                        type = CommitPoint.Type.valueOf(parser.text());
-                    }
-                }
-            }
-
-            if (version == -1) {
-                throw new IOException("Malformed commit, missing version");
-            }
-            if (name == null) {
-                throw new IOException("Malformed commit, missing name");
-            }
-            if (type == null) {
-                throw new IOException("Malformed commit, missing type");
-            }
-
-            return new CommitPoint(version, name, type, indexFiles, translogFiles);
-        }
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/shard/CommitPointsTests.java b/core/src/test/java/org/elasticsearch/index/shard/CommitPointsTests.java
deleted file mode 100644
index 6fd66f3fb73..00000000000
--- a/core/src/test/java/org/elasticsearch/index/shard/CommitPointsTests.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.shard;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.test.ESTestCase;
-
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-
-public class CommitPointsTests extends ESTestCase {
-    private final Logger logger = Loggers.getLogger(CommitPointsTests.class);
-
-    public void testCommitPointXContent() throws Exception {
-        ArrayList indexFiles = new ArrayList<>();
-        indexFiles.add(new CommitPoint.FileInfo("file1", "file1_p", 100, "ck1"));
-        indexFiles.add(new CommitPoint.FileInfo("file2", "file2_p", 200, "ck2"));
-
-        ArrayList translogFiles = new ArrayList<>();
-        translogFiles.add(new CommitPoint.FileInfo("t_file1", "t_file1_p", 100, null));
-        translogFiles.add(new CommitPoint.FileInfo("t_file2", "t_file2_p", 200, null));
-
-        CommitPoint commitPoint = new CommitPoint(1, "test", CommitPoint.Type.GENERATED, indexFiles, translogFiles);
-
-        byte[] serialized = CommitPoints.toXContent(commitPoint);
-        logger.info("serialized commit_point {}", new String(serialized, StandardCharsets.UTF_8));
-
-        CommitPoint desCp = CommitPoints.fromXContent(serialized);
-        assertThat(desCp.version(), equalTo(commitPoint.version()));
-        assertThat(desCp.name(), equalTo(commitPoint.name()));
-
-        assertThat(desCp.indexFiles().size(), equalTo(commitPoint.indexFiles().size()));
-        for (int i = 0; i < desCp.indexFiles().size(); i++) {
-            assertThat(desCp.indexFiles().get(i).name(), equalTo(commitPoint.indexFiles().get(i).name()));
-            assertThat(desCp.indexFiles().get(i).physicalName(), equalTo(commitPoint.indexFiles().get(i).physicalName()));
-            assertThat(desCp.indexFiles().get(i).length(), equalTo(commitPoint.indexFiles().get(i).length()));
-            assertThat(desCp.indexFiles().get(i).checksum(), equalTo(commitPoint.indexFiles().get(i).checksum()));
-        }
-
-        assertThat(desCp.translogFiles().size(), equalTo(commitPoint.translogFiles().size()));
-        for (int i = 0; i < desCp.indexFiles().size(); i++) {
-            assertThat(desCp.translogFiles().get(i).name(), equalTo(commitPoint.translogFiles().get(i).name()));
-            assertThat(desCp.translogFiles().get(i).physicalName(), equalTo(commitPoint.translogFiles().get(i).physicalName()));
-            assertThat(desCp.translogFiles().get(i).length(), equalTo(commitPoint.translogFiles().get(i).length()));
-            assertThat(desCp.translogFiles().get(i).checksum(), nullValue());
-        }
-    }
-}

From 816878bd4d6f416751ba9d15f3ea869ca7efd598 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?=
 <10398885+cbuescher@users.noreply.github.com>
Date: Fri, 8 Dec 2017 10:11:57 +0100
Subject: [PATCH 212/297] [Tests] Add test for
 GeoShapeFieldType#setStrategyName (#27703)

---
 .../index/mapper/GeoShapeFieldTypeTests.java  | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java
index 41ae4bdfbe4..a1c225f8a06 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldTypeTests.java
@@ -18,11 +18,13 @@
  */
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.common.geo.SpatialStrategy;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
-import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.GeoShapeFieldMapper.GeoShapeFieldType;
 import org.junit.Before;
 
+import java.io.IOException;
+
 public class GeoShapeFieldTypeTests extends FieldTypeTestCase {
     @Override
     protected MappedFieldType createDefaultFieldType() {
@@ -68,4 +70,17 @@ public class GeoShapeFieldTypeTests extends FieldTypeTestCase {
             }
         });
     }
+
+    /**
+     * Test for {@link GeoShapeFieldType#setStrategyName(String)} that checks that {@link GeoShapeFieldType#pointsOnly()}
+     * gets set as a side effect when using SpatialStrategy.TERM
+     */
+    public void testSetStrategyName() throws IOException {
+        GeoShapeFieldType fieldType = new GeoShapeFieldMapper.GeoShapeFieldType();
+        assertFalse(fieldType.pointsOnly());
+        fieldType.setStrategyName(SpatialStrategy.RECURSIVE.getStrategyName());
+        assertFalse(fieldType.pointsOnly());
+        fieldType.setStrategyName(SpatialStrategy.TERM.getStrategyName());
+        assertTrue(fieldType.pointsOnly());
+    }
 }

From 27e157f67c2ccaf9c3b8eda3980eaf26e0f1dd7b Mon Sep 17 00:00:00 2001
From: javanna 
Date: Fri, 8 Dec 2017 10:51:10 +0100
Subject: [PATCH 213/297] [TEST] remove code duplications in RequestTests

---
 .../elasticsearch/client/RequestTests.java    | 104 ++++++------------
 1 file changed, 35 insertions(+), 69 deletions(-)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
index f39a7c77b80..7d63ed1ed40 100755
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
@@ -42,7 +42,6 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
-import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
 import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.common.CheckedBiConsumer;
@@ -156,9 +155,9 @@ public class RequestTests extends ESTestCase {
         Map expectedParams = new HashMap<>();
 
         setRandomTimeout(deleteRequest::timeout, ReplicationRequest.DEFAULT_TIMEOUT, expectedParams);
-        setRandomRefreshPolicy(deleteRequest, expectedParams);
+        setRandomRefreshPolicy(deleteRequest::setRefreshPolicy, expectedParams);
         setRandomVersion(deleteRequest, expectedParams);
-        setRandomVersionType(deleteRequest, expectedParams);
+        setRandomVersionType(deleteRequest::versionType, expectedParams);
 
         if (frequently()) {
             if (randomBoolean()) {
@@ -223,27 +222,13 @@ public class RequestTests extends ESTestCase {
                     expectedParams.put("version", Long.toString(version));
                 }
             }
-            if (randomBoolean()) {
-                VersionType versionType = randomFrom(VersionType.values());
-                getRequest.versionType(versionType);
-                if (versionType != VersionType.INTERNAL) {
-                    expectedParams.put("version_type", versionType.name().toLowerCase(Locale.ROOT));
-                }
-            }
+            setRandomVersionType(getRequest::versionType, expectedParams);
             if (randomBoolean()) {
                 int numStoredFields = randomIntBetween(1, 10);
                 String[] storedFields = new String[numStoredFields];
-                StringBuilder storedFieldsParam = new StringBuilder();
-                for (int i = 0; i < numStoredFields; i++) {
-                    String storedField = randomAlphaOfLengthBetween(3, 10);
-                    storedFields[i] = storedField;
-                    storedFieldsParam.append(storedField);
-                    if (i < numStoredFields - 1) {
-                        storedFieldsParam.append(",");
-                    }
-                }
+                String storedFieldsParam = randomFields(storedFields);
                 getRequest.storedFields(storedFields);
-                expectedParams.put("stored_fields", storedFieldsParam.toString());
+                expectedParams.put("stored_fields", storedFieldsParam);
             }
             if (randomBoolean()) {
                 randomizeFetchSourceContextParams(getRequest::fetchSourceContext, expectedParams);
@@ -350,7 +335,7 @@ public class RequestTests extends ESTestCase {
         }
 
         setRandomTimeout(indexRequest::timeout, ReplicationRequest.DEFAULT_TIMEOUT, expectedParams);
-        setRandomRefreshPolicy(indexRequest, expectedParams);
+        setRandomRefreshPolicy(indexRequest::setRefreshPolicy, expectedParams);
 
         // There is some logic around _create endpoint and version/version type
         if (indexRequest.opType() == DocWriteRequest.OpType.CREATE) {
@@ -358,7 +343,7 @@ public class RequestTests extends ESTestCase {
             expectedParams.put("version", Long.toString(Versions.MATCH_DELETED));
         } else {
             setRandomVersion(indexRequest, expectedParams);
-            setRandomVersionType(indexRequest, expectedParams);
+            setRandomVersionType(indexRequest::versionType, expectedParams);
         }
 
         if (frequently()) {
@@ -462,20 +447,8 @@ public class RequestTests extends ESTestCase {
             }
         }
         setRandomWaitForActiveShards(updateRequest::waitForActiveShards, expectedParams);
-        if (randomBoolean()) {
-            long version = randomLong();
-            updateRequest.version(version);
-            if (version != Versions.MATCH_ANY) {
-                expectedParams.put("version", Long.toString(version));
-            }
-        }
-        if (randomBoolean()) {
-            VersionType versionType = randomFrom(VersionType.values());
-            updateRequest.versionType(versionType);
-            if (versionType != VersionType.INTERNAL) {
-                expectedParams.put("version_type", versionType.name().toLowerCase(Locale.ROOT));
-            }
-        }
+        setRandomVersion(updateRequest, expectedParams);
+        setRandomVersionType(updateRequest::versionType, expectedParams);
         if (randomBoolean()) {
             int retryOnConflict = randomIntBetween(0, 5);
             updateRequest.retryOnConflict(retryOnConflict);
@@ -519,7 +492,7 @@ public class RequestTests extends ESTestCase {
         }
     }
 
-    public void testUpdateWithDifferentContentTypes() throws IOException {
+    public void testUpdateWithDifferentContentTypes() {
         IllegalStateException exception = expectThrows(IllegalStateException.class, () -> {
             UpdateRequest updateRequest = new UpdateRequest();
             updateRequest.doc(new IndexRequest().source(singletonMap("field", "doc"), XContentType.JSON));
@@ -542,13 +515,7 @@ public class RequestTests extends ESTestCase {
             expectedParams.put("timeout", BulkShardRequest.DEFAULT_TIMEOUT.getStringRep());
         }
 
-        if (randomBoolean()) {
-            WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
-            bulkRequest.setRefreshPolicy(refreshPolicy);
-            if (refreshPolicy != WriteRequest.RefreshPolicy.NONE) {
-                expectedParams.put("refresh", refreshPolicy.getValue());
-            }
-        }
+        setRandomRefreshPolicy(bulkRequest::setRefreshPolicy, expectedParams);
 
         XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
 
@@ -561,7 +528,7 @@ public class RequestTests extends ESTestCase {
             BytesReference source = RandomObjects.randomSource(random(), xContentType);
             DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
 
-            DocWriteRequest docWriteRequest = null;
+            DocWriteRequest docWriteRequest;
             if (opType == DocWriteRequest.OpType.INDEX) {
                 IndexRequest indexRequest = new IndexRequest(index, type, id).source(source, xContentType);
                 docWriteRequest = indexRequest;
@@ -591,6 +558,8 @@ public class RequestTests extends ESTestCase {
                 }
             } else if (opType == DocWriteRequest.OpType.DELETE) {
                 docWriteRequest = new DeleteRequest(index, type, id);
+            } else {
+                throw new UnsupportedOperationException("optype [" + opType + "] not supported");
             }
 
             if (randomBoolean()) {
@@ -995,31 +964,15 @@ public class RequestTests extends ESTestCase {
             } else {
                 int numIncludes = randomIntBetween(0, 5);
                 String[] includes = new String[numIncludes];
-                StringBuilder includesParam = new StringBuilder();
-                for (int i = 0; i < numIncludes; i++) {
-                    String include = randomAlphaOfLengthBetween(3, 10);
-                    includes[i] = include;
-                    includesParam.append(include);
-                    if (i < numIncludes - 1) {
-                        includesParam.append(",");
-                    }
-                }
+                String includesParam = randomFields(includes);
                 if (numIncludes > 0) {
-                    expectedParams.put("_source_include", includesParam.toString());
+                    expectedParams.put("_source_include", includesParam);
                 }
                 int numExcludes = randomIntBetween(0, 5);
                 String[] excludes = new String[numExcludes];
-                StringBuilder excludesParam = new StringBuilder();
-                for (int i = 0; i < numExcludes; i++) {
-                    String exclude = randomAlphaOfLengthBetween(3, 10);
-                    excludes[i] = exclude;
-                    excludesParam.append(exclude);
-                    if (i < numExcludes - 1) {
-                        excludesParam.append(",");
-                    }
-                }
+                String excludesParam = randomFields(excludes);
                 if (numExcludes > 0) {
-                    expectedParams.put("_source_exclude", excludesParam.toString());
+                    expectedParams.put("_source_exclude", excludesParam);
                 }
                 consumer.accept(new FetchSourceContext(true, includes, excludes));
             }
@@ -1074,10 +1027,10 @@ public class RequestTests extends ESTestCase {
         }
     }
 
-    private static void setRandomRefreshPolicy(ReplicatedWriteRequest request, Map expectedParams) {
+    private static void setRandomRefreshPolicy(Consumer setter, Map expectedParams) {
         if (randomBoolean()) {
             WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
-            request.setRefreshPolicy(refreshPolicy);
+            setter.accept(refreshPolicy);
             if (refreshPolicy != WriteRequest.RefreshPolicy.NONE) {
                 expectedParams.put("refresh", refreshPolicy.getValue());
             }
@@ -1094,13 +1047,26 @@ public class RequestTests extends ESTestCase {
         }
     }
 
-    private static void setRandomVersionType(DocWriteRequest request, Map expectedParams) {
+    private static void setRandomVersionType(Consumer setter, Map expectedParams) {
         if (randomBoolean()) {
             VersionType versionType = randomFrom(VersionType.values());
-            request.versionType(versionType);
+            setter.accept(versionType);
             if (versionType != VersionType.INTERNAL) {
                 expectedParams.put("version_type", versionType.name().toLowerCase(Locale.ROOT));
             }
         }
     }
+
+    private static String randomFields(String[] fields) {
+        StringBuilder excludesParam = new StringBuilder();
+        for (int i = 0; i < fields.length; i++) {
+            String exclude = randomAlphaOfLengthBetween(3, 10);
+            fields[i] = exclude;
+            excludesParam.append(exclude);
+            if (i < fields.length - 1) {
+                excludesParam.append(",");
+            }
+        }
+        return excludesParam.toString();
+    }
 }

From 952c859f528188f40815fccc3e94238d68a1fb19 Mon Sep 17 00:00:00 2001
From: Simon Willnauer 
Date: Fri, 8 Dec 2017 12:28:27 +0100
Subject: [PATCH 214/297] Test out of order delivery of append only index and
 retry with an intermediate delete

---
 .../index/engine/InternalEngineTests.java     | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 1b700b80086..8a133f35c24 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -2983,6 +2983,50 @@ public class InternalEngineTests extends EngineTestCase {
         }
     }
 
+    public void testDoubleDeliveryReplicaAppendingAndDeleteOnly() throws IOException {
+        final ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(),
+            new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        Engine.Index operation = appendOnlyReplica(doc, false, 1, randomIntBetween(0, 5));
+        Engine.Index retry = appendOnlyReplica(doc, true, 1, randomIntBetween(0, 5));
+        Engine.Delete delete = new Engine.Delete(operation.type(), operation.id(), operation.uid(),
+            Math.max(retry.seqNo(), operation.seqNo())+1, operation.primaryTerm(), operation.version()+1, operation.versionType(),
+            REPLICA, operation.startTime()+1);
+        // operations with a seq# equal or lower to the local checkpoint are not indexed to lucene
+        // and the version lookup is skipped
+        final boolean belowLckp = operation.seqNo() == 0 && retry.seqNo() == 0;
+        if (randomBoolean()) {
+            Engine.IndexResult indexResult = engine.index(operation);
+            assertFalse(engine.indexWriterHasDeletions());
+            assertEquals(0, engine.getNumVersionLookups());
+            assertNotNull(indexResult.getTranslogLocation());
+            engine.delete(delete);
+            assertEquals(1, engine.getNumVersionLookups());
+            assertTrue(engine.indexWriterHasDeletions());
+            Engine.IndexResult retryResult = engine.index(retry);
+            assertEquals(belowLckp ? 1 : 2, engine.getNumVersionLookups());
+            assertNotNull(retryResult.getTranslogLocation());
+            assertTrue(retryResult.getTranslogLocation().compareTo(indexResult.getTranslogLocation()) > 0);
+        } else {
+            Engine.IndexResult retryResult = engine.index(retry);
+            assertFalse(engine.indexWriterHasDeletions());
+            assertEquals(1, engine.getNumVersionLookups());
+            assertNotNull(retryResult.getTranslogLocation());
+            engine.delete(delete);
+            assertTrue(engine.indexWriterHasDeletions());
+            assertEquals(2, engine.getNumVersionLookups());
+            Engine.IndexResult indexResult = engine.index(operation);
+            assertEquals(belowLckp ? 2 : 3, engine.getNumVersionLookups());
+            assertNotNull(retryResult.getTranslogLocation());
+            assertTrue(retryResult.getTranslogLocation().compareTo(indexResult.getTranslogLocation()) < 0);
+        }
+
+        engine.refresh("test");
+        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
+            TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
+            assertEquals(0, topDocs.totalHits);
+        }
+    }
+
     public void testDoubleDeliveryReplicaAppendingOnly() throws IOException {
         final ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(),
             new BytesArray("{}".getBytes(Charset.defaultCharset())), null);

From 8f104cc08c46a00233c61abe84eaed301b7a9526 Mon Sep 17 00:00:00 2001
From: Simon Willnauer 
Date: Fri, 8 Dec 2017 12:35:02 +0100
Subject: [PATCH 215/297] [TEST] Now actually wait for merges

Relates to #27651
---
 .../java/org/elasticsearch/index/shard/IndexShardTests.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index ebf39b7e089..e4b1eb083b5 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -2856,7 +2856,7 @@ public class IndexShardTests extends IndexShardTestCase {
 
         // We need to wait for all ongoing merges to complete. The reason is that during a merge the
         // IndexWriter holds the core cache key open and causes the memory to be registered in the breaker
-        primary.forceMerge(new ForceMergeRequest());
+        primary.forceMerge(new ForceMergeRequest().maxNumSegments(1).flush(true));
 
         // Close remaining searchers
         IOUtils.close(searchers);

From 58b4d6c5fc0ac9b6b42d6606424b4949a961f740 Mon Sep 17 00:00:00 2001
From: Avneesh Chadha 
Date: Fri, 8 Dec 2017 19:01:45 +0530
Subject: [PATCH 216/297] [Issue-27716]: CONTRIBUTING.md IntelliJ
 configurations settings are confusing. (#27717)

* Improved paragraph describing how to run unit tests from Intellij.
* Added information about how to run a local copy of elasticsearch from the source.
---
 CONTRIBUTING.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6bbb655a18b..5e81b188288 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -119,11 +119,13 @@ Alternatively, `idea.no.launcher=true` can be set in the
 [`idea.properties`](https://www.jetbrains.com/help/idea/file-idea-properties.html)
 file which can be accessed under Help > Edit Custom Properties (this will require a
 restart of IDEA). For IDEA 2017.3 and above, in addition to the JVM option, you will need to go to
-`Run->Edit Configurations...` and change the value for the `Shorten command line` setting from
+`Run->Edit Configurations->...->Defaults->JUnit` and change the value for the `Shorten command line` setting from
 `user-local default: none` to `classpath file`. You may also need to [remove `ant-javafx.jar` from your
 classpath](https://github.com/elastic/elasticsearch/issues/14348) if that is
 reported as a source of jar hell.
 
+To run an instance of elasticsearch from the source code run `gradle run`
+
 The Elasticsearch codebase makes heavy use of Java `assert`s and the
 test runner requires that assertions be enabled within the JVM. This
 can be accomplished by passing the flag `-ea` to the JVM on startup.

From 3d3a1d2a0dd2c5ee39611e5e47c7d6fcabcd528a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?=
 <10398885+cbuescher@users.noreply.github.com>
Date: Fri, 8 Dec 2017 15:12:15 +0100
Subject: [PATCH 217/297] Adding short description for experimental status in
 docs

---
 docs/reference/search/rank-eval.asciidoc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc
index 0e9e8a821ac..5a3277fa43b 100644
--- a/docs/reference/search/rank-eval.asciidoc
+++ b/docs/reference/search/rank-eval.asciidoc
@@ -1,14 +1,15 @@
 [[search-rank-eval]]
 == Ranking Evaluation API
 
-experimental[]
-
 The ranking evaluation API allows to evaluate the quality of ranked search
 results over a set of typical search queries. Given this set of queries and a
 list or manually rated documents, the `_rank_eval` endpoint calculates and
 returns typical information retrieval metrics like _mean reciprocal rank_,
 _precision_ or _discounted cumulative gain_.
 
+experimental[The ranking evaluation API is new and may change in non-backwards compatible ways in the future,
+even on minor versions updates.]
+
 === Overview
 
 Search quality evaluation starts with looking at the users of your search application, and the things that they are searching for.

From 9b9f85e5098f01cd063e54c7058aa49e66afd15d Mon Sep 17 00:00:00 2001
From: David Roberts 
Date: Fri, 8 Dec 2017 14:29:06 +0000
Subject: [PATCH 218/297] Add missing 's' to tmpdir name (#27721)

When using mktemp from coreutils there was an 's' missing from
elasticsearch.

Follow-up for #27659
---
 distribution/src/main/resources/bin/elasticsearch-env | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch-env b/distribution/src/main/resources/bin/elasticsearch-env
index 83380400173..a5cf04da77d 100644
--- a/distribution/src/main/resources/bin/elasticsearch-env
+++ b/distribution/src/main/resources/bin/elasticsearch-env
@@ -80,7 +80,7 @@ if [ -z "$ES_TMPDIR" ]; then
   mktemp_coreutils=$?
   set -e
   if [ $mktemp_coreutils -eq 0 ]; then
-    ES_TMPDIR=`mktemp -d --tmpdir "elasticearch.XXXXXXXX"`
+    ES_TMPDIR=`mktemp -d --tmpdir "elasticsearch.XXXXXXXX"`
   else
     ES_TMPDIR=`mktemp -d -t elasticsearch`
   fi

From 64dd21af11b7a8f2df6f2abd44ccb04526b0a290 Mon Sep 17 00:00:00 2001
From: Boaz Leskes 
Date: Fri, 8 Dec 2017 16:45:45 +0100
Subject: [PATCH 219/297] remove await fix from
 FullClusterRestartIT.testRecovery

---
 .../java/org/elasticsearch/upgrades/FullClusterRestartIT.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
index d357a228c43..5c61cacd4e9 100644
--- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -615,7 +615,6 @@ public class FullClusterRestartIT extends ESRestTestCase {
      * Tests recovery of an index with or without a translog and the
      * statistics we gather about that.
      */
-    @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/27649")
     public void testRecovery() throws IOException {
         int count;
         boolean shouldHaveTranslog;

From ec5e540174711966895704c38f6f44d4df8735fd Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Fri, 8 Dec 2017 11:23:24 -0500
Subject: [PATCH 220/297] Fix routing with leading or trailing whitespace

The problem here is that splitting was using a method that intentionally
trims whitespace (the method is really meant to be used for splitting
parameters where whitespace should be trimmed like list
settings). However, for routing values whitespace should not be trimmed
because we allow routing with leading and trailing spaces. This commit
switches the parsing of these routing values to a method that does not
trim whitespace.

Relates #27712
---
 .../resources/checkstyle_suppressions.xml     |  1 -
 .../cluster/metadata/AliasMetaData.java       |  5 +-
 .../metadata/IndexNameExpressionResolver.java |  8 ++-
 .../cluster/metadata/AliasMetaDataTests.java  | 55 ++++++++++++++++
 .../routing/AliasResolveRoutingIT.java        | 65 ++++++++++++++++---
 5 files changed, 120 insertions(+), 14 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 54dfe661f35..67302f1fe8f 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -658,7 +658,6 @@
   
   
   
-  
   
   
   
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
index 8071871fbfe..c0262a6d01d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -58,7 +59,7 @@ public class AliasMetaData extends AbstractDiffable {
         this.indexRouting = indexRouting;
         this.searchRouting = searchRouting;
         if (searchRouting != null) {
-            searchRoutingValues = Collections.unmodifiableSet(Strings.splitStringByCommaToSet(searchRouting));
+            searchRoutingValues = Collections.unmodifiableSet(Sets.newHashSet(Strings.splitStringByCommaToArray(searchRouting)));
         } else {
             searchRoutingValues = emptySet();
         }
@@ -186,7 +187,7 @@ public class AliasMetaData extends AbstractDiffable {
         }
         if (in.readBoolean()) {
             searchRouting = in.readString();
-            searchRoutingValues = Collections.unmodifiableSet(Strings.splitStringByCommaToSet(searchRouting));
+            searchRoutingValues = Collections.unmodifiableSet(Sets.newHashSet(Strings.splitStringByCommaToArray(searchRouting)));
         } else {
             searchRouting = null;
             searchRoutingValues = emptySet();
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 6dc92a44bb0..1f36e50ca1d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.joda.DateMathParser;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.IndexClosedException;
@@ -358,6 +359,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
             resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions);
         }
 
+        // TODO: it appears that this can never be true?
         if (isAllIndices(resolvedExpressions)) {
             return resolveSearchRoutingAllIndices(state.metaData(), routing);
         }
@@ -367,7 +369,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
         // List of indices that don't require any routing
         Set norouting = new HashSet<>();
         if (routing != null) {
-            paramRouting = Strings.splitStringByCommaToSet(routing);
+            paramRouting = Sets.newHashSet(Strings.splitStringByCommaToArray(routing));
         }
 
         for (String expression : resolvedExpressions) {
@@ -442,9 +444,9 @@ public class IndexNameExpressionResolver extends AbstractComponent {
     /**
      * Sets the same routing for all indices
      */
-    private Map> resolveSearchRoutingAllIndices(MetaData metaData, String routing) {
+    public Map> resolveSearchRoutingAllIndices(MetaData metaData, String routing) {
         if (routing != null) {
-            Set r = Strings.splitStringByCommaToSet(routing);
+            Set r = Sets.newHashSet(Strings.splitStringByCommaToArray(routing));
             Map> routings = new HashMap<>();
             String[] concreteIndices = metaData.getConcreteAllIndices();
             for (String index : concreteIndices) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java
new file mode 100644
index 00000000000..d5ae07f4341
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/AliasMetaDataTests.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AliasMetaDataTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        final AliasMetaData before =
+                AliasMetaData
+                        .builder("alias")
+                        .filter("{ \"term\": \"foo\"}")
+                        .indexRouting("indexRouting")
+                        .routing("routing")
+                        .searchRouting("trim,tw , ltw , lw")
+                        .build();
+
+        assertThat(before.searchRoutingValues(), equalTo(Sets.newHashSet("trim", "tw ", " ltw ", " lw")));
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        before.writeTo(out);
+
+        final StreamInput in = out.bytes().streamInput();
+        final AliasMetaData after = new AliasMetaData(in);
+
+        assertThat(after, equalTo(before));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java
index c90ce51e48e..7a0ce93e77d 100644
--- a/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java
+++ b/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java
@@ -26,13 +26,13 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.test.ESIntegTestCase;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.equalTo;
@@ -40,7 +40,6 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class AliasResolveRoutingIT extends ESIntegTestCase {
 
-
     // see https://github.com/elastic/elasticsearch/issues/13278
     public void testSearchClosedWildcardIndex() throws ExecutionException, InterruptedException {
         createIndex("test-0");
@@ -52,10 +51,17 @@ public class AliasResolveRoutingIT extends ESIntegTestCase {
             client().prepareIndex("test-0", "type1", "2").setSource("field1", "quick brown"),
             client().prepareIndex("test-0", "type1", "3").setSource("field1", "quick"));
         refresh("test-*");
-        assertHitCount(client().prepareSearch().setIndices("alias-*").setIndicesOptions(IndicesOptions.lenientExpandOpen()).setQuery(queryStringQuery("quick")).get(), 3L);
+        assertHitCount(
+                client()
+                        .prepareSearch()
+                        .setIndices("alias-*")
+                        .setIndicesOptions(IndicesOptions.lenientExpandOpen())
+                        .setQuery(queryStringQuery("quick"))
+                        .get(),
+                3L);
     }
 
-    public void testResolveIndexRouting() throws Exception {
+    public void testResolveIndexRouting() {
         createIndex("test1");
         createIndex("test2");
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
@@ -97,9 +103,10 @@ public class AliasResolveRoutingIT extends ESIntegTestCase {
         }
     }
 
-    public void testResolveSearchRouting() throws Exception {
+    public void testResolveSearchRouting() {
         createIndex("test1");
         createIndex("test2");
+        createIndex("test3");
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
 
         client().admin().indices().prepareAliases()
@@ -108,7 +115,10 @@ public class AliasResolveRoutingIT extends ESIntegTestCase {
                 .addAliasAction(AliasActions.add().index("test2").alias("alias20").routing("0"))
                 .addAliasAction(AliasActions.add().index("test2").alias("alias21").routing("1"))
                 .addAliasAction(AliasActions.add().index("test1").alias("alias0").routing("0"))
-                .addAliasAction(AliasActions.add().index("test2").alias("alias0").routing("0")).get();
+                .addAliasAction(AliasActions.add().index("test2").alias("alias0").routing("0"))
+                .addAliasAction(AliasActions.add().index("test3").alias("alias3tw").routing("tw "))
+                .addAliasAction(AliasActions.add().index("test3").alias("alias3ltw").routing(" ltw "))
+                .addAliasAction(AliasActions.add().index("test3").alias("alias3lw").routing(" lw")).get();
 
         ClusterState state = clusterService().state();
         IndexNameExpressionResolver indexNameExpressionResolver = internalCluster().getInstance(IndexNameExpressionResolver.class);
@@ -118,7 +128,9 @@ public class AliasResolveRoutingIT extends ESIntegTestCase {
         assertThat(indexNameExpressionResolver.resolveSearchRouting(state, null, "alias10"), equalTo(newMap("test1", newSet("0"))));
         assertThat(indexNameExpressionResolver.resolveSearchRouting(state, "0", "alias10"), equalTo(newMap("test1", newSet("0"))));
         assertThat(indexNameExpressionResolver.resolveSearchRouting(state, "1", "alias10"), nullValue());
-        assertThat(indexNameExpressionResolver.resolveSearchRouting(state, null, "alias0"), equalTo(newMap("test1", newSet("0"), "test2", newSet("0"))));
+        assertThat(
+                indexNameExpressionResolver.resolveSearchRouting(state, null, "alias0"),
+                equalTo(newMap("test1", newSet("0"), "test2", newSet("0"))));
 
         assertThat(indexNameExpressionResolver.resolveSearchRouting(state, null, new String[]{"alias10", "alias20"}),
                 equalTo(newMap("test1", newSet("0"), "test2", newSet("0"))));
@@ -143,13 +155,42 @@ public class AliasResolveRoutingIT extends ESIntegTestCase {
                 equalTo(newMap("test1", newSet("0"), "test2", newSet("1"))));
         assertThat(indexNameExpressionResolver.resolveSearchRouting(state, "0,1,2", new String[]{"test1", "alias10", "alias21"}),
                 equalTo(newMap("test1", newSet("0", "1", "2"), "test2", newSet("1"))));
+
+        assertThat(
+                indexNameExpressionResolver.resolveSearchRouting(state, "tw , ltw , lw", "test1"),
+                equalTo(newMap("test1", newSet("tw ", " ltw ", " lw"))));
+        assertThat(
+                indexNameExpressionResolver.resolveSearchRouting(state, "tw , ltw , lw", "alias3tw"),
+                equalTo(newMap("test3", newSet("tw "))));
+        assertThat(
+                indexNameExpressionResolver.resolveSearchRouting(state, "tw , ltw , lw", "alias3ltw"),
+                equalTo(newMap("test3", newSet(" ltw "))));
+        assertThat(
+                indexNameExpressionResolver.resolveSearchRouting(state, "tw , ltw , lw", "alias3lw"),
+                equalTo(newMap("test3", newSet(" lw"))));
+        assertThat(
+                indexNameExpressionResolver.resolveSearchRouting(state, "0,tw , ltw , lw", "test1", "alias3ltw"),
+                equalTo(newMap("test1", newSet("0", "tw ", " ltw ", " lw"), "test3", newSet(" ltw "))));
+
+        assertThat(
+                indexNameExpressionResolver.resolveSearchRouting(state, "0,1,2,tw , ltw , lw", (String[])null),
+                equalTo(newMap(
+                        "test1", newSet("0", "1", "2", "tw ", " ltw ", " lw"),
+                        "test2", newSet("0", "1", "2", "tw ", " ltw ", " lw"),
+                        "test3", newSet("0", "1", "2", "tw ", " ltw ", " lw"))));
+
+        assertThat(
+                indexNameExpressionResolver.resolveSearchRoutingAllIndices(state.metaData(), "0,1,2,tw , ltw , lw"),
+                equalTo(newMap(
+                        "test1", newSet("0", "1", "2", "tw ", " ltw ", " lw"),
+                        "test2", newSet("0", "1", "2", "tw ", " ltw ", " lw"),
+                        "test3", newSet("0", "1", "2", "tw ", " ltw ", " lw"))));
     }
 
     private  Set newSet(T... elements) {
         return newHashSet(elements);
     }
 
-
     private  Map newMap(K key, V value) {
         Map r = new HashMap<>();
         r.put(key, value);
@@ -163,4 +204,12 @@ public class AliasResolveRoutingIT extends ESIntegTestCase {
         return r;
     }
 
+    private  Map newMap(K key1, V value1, K key2, V value2, K key3, V value3) {
+        Map r = new HashMap<>();
+        r.put(key1, value1);
+        r.put(key2, value2);
+        r.put(key3, value3);
+        return r;
+    }
+
 }

From ad8a571677c1c0b4ecb5b2a3da0debf1cfda8906 Mon Sep 17 00:00:00 2001
From: Tim Brooks 
Date: Fri, 8 Dec 2017 09:32:09 -0700
Subject: [PATCH 221/297] Add read timeouts to http module (#27713)

We currently do not have any server-side read timeouts implemented in
elasticsearch. This commit adds a read timeout setting that defaults to
30 seconds. If after 30 seconds a read has not occurred, the channel
will be closed. A timeout of value of 0 will disable the timeout.
---
 .../http/HttpTransportSettings.java           |  5 ++
 .../netty4/Netty4HttpServerTransport.java     |  8 ++-
 .../Netty4HttpServerTransportTests.java       | 59 +++++++++++++++++++
 3 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
index 54be8b4ecd7..cf9bd3a42ee 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
@@ -26,8 +26,10 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.transport.PortsRange;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
 
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 
 import static java.util.Collections.emptyList;
@@ -93,6 +95,9 @@ public final class HttpTransportSettings {
     public static final Setting SETTING_HTTP_RESET_COOKIES =
         Setting.boolSetting("http.reset_cookies", false, Property.NodeScope);
 
+    public static final Setting SETTING_HTTP_READ_TIMEOUT =
+        Setting.timeSetting("http.read_timeout", new TimeValue(30, TimeUnit.SECONDS), new TimeValue(0), Property.NodeScope);
+
     public static final Setting SETTING_HTTP_TCP_NO_DELAY =
         boolSetting("http.tcp_no_delay", NetworkService.TCP_NO_DELAY, Setting.Property.NodeScope);
     public static final Setting SETTING_HTTP_TCP_KEEP_ALIVE =
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
index 893efeb6957..31b32a8ab94 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
@@ -40,6 +40,7 @@ import io.netty.handler.codec.http.HttpObjectAggregator;
 import io.netty.handler.codec.http.HttpRequestDecoder;
 import io.netty.handler.codec.http.HttpResponseEncoder;
 import io.netty.handler.timeout.ReadTimeoutException;
+import io.netty.handler.timeout.ReadTimeoutHandler;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.common.Strings;
@@ -86,7 +87,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.common.settings.Setting.boolSetting;
 import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
@@ -105,6 +105,7 @@ import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_INIT
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PORT;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_RESET_COOKIES;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_TCP_KEEP_ALIVE;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_TCP_NO_DELAY;
@@ -172,6 +173,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
     protected final ByteSizeValue tcpSendBufferSize;
     protected final ByteSizeValue tcpReceiveBufferSize;
     protected final RecvByteBufAllocator recvByteBufAllocator;
+    private final int readTimeoutMillis;
 
     protected final int maxCompositeBufferComponents;
     private final Dispatcher dispatcher;
@@ -220,6 +222,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
         this.tcpSendBufferSize = SETTING_HTTP_TCP_SEND_BUFFER_SIZE.get(settings);
         this.tcpReceiveBufferSize = SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings);
         this.detailedErrorsEnabled = SETTING_HTTP_DETAILED_ERRORS_ENABLED.get(settings);
+        this.readTimeoutMillis = Math.toIntExact(SETTING_HTTP_READ_TIMEOUT.get(settings).getMillis());
 
         ByteSizeValue receivePredictor = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE.get(settings);
         recvByteBufAllocator = new FixedRecvByteBufAllocator(receivePredictor.bytesAsInt());
@@ -480,7 +483,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
     protected void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
         if (cause instanceof ReadTimeoutException) {
             if (logger.isTraceEnabled()) {
-                logger.trace("Connection timeout [{}]", ctx.channel().remoteAddress());
+                logger.trace("Read timeout [{}]", ctx.channel().remoteAddress());
             }
             ctx.channel().close();
         } else {
@@ -524,6 +527,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
         @Override
         protected void initChannel(Channel ch) throws Exception {
             ch.pipeline().addLast("openChannels", transport.serverOpenChannels);
+            ch.pipeline().addLast("read_timeout", new ReadTimeoutHandler(transport.readTimeoutMillis, TimeUnit.MILLISECONDS));
             final HttpRequestDecoder decoder = new HttpRequestDecoder(
                 Math.toIntExact(transport.maxInitialLineLength.getBytes()),
                 Math.toIntExact(transport.maxHeaderSize.getBytes()),
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
index 846c59565c2..c347bd2805d 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
@@ -19,8 +19,15 @@
 
 package org.elasticsearch.http.netty4;
 
+import io.netty.bootstrap.Bootstrap;
 import io.netty.buffer.ByteBufUtil;
 import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerAdapter;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioSocketChannel;
 import io.netty.handler.codec.TooLongFrameException;
 import io.netty.handler.codec.http.DefaultFullHttpRequest;
 import io.netty.handler.codec.http.FullHttpRequest;
@@ -39,6 +46,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.BindHttpException;
@@ -63,6 +71,8 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
@@ -313,4 +323,53 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
             assertNull(threadPool.getThreadContext().getTransient("bar_bad"));
         }
     }
+
+    public void testReadTimeout() throws Exception {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                throw new AssertionError("Should not have received a dispatched request");
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestRequest request,
+                                           final RestChannel channel,
+                                           final ThreadContext threadContext,
+                                           final Throwable cause) {
+                throw new AssertionError("Should not have received a dispatched request");
+            }
+
+        };
+
+        Settings settings = Settings.builder()
+            .put(HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT.getKey(), new TimeValue(randomIntBetween(100, 300)))
+            .build();
+
+
+        NioEventLoopGroup group = new NioEventLoopGroup();
+        try (Netty4HttpServerTransport transport =
+                 new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(), dispatcher)) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress.boundAddresses());
+
+            AtomicBoolean channelClosed = new AtomicBoolean(false);
+
+            Bootstrap clientBootstrap = new Bootstrap().channel(NioSocketChannel.class).handler(new ChannelInitializer() {
+
+                @Override
+                protected void initChannel(SocketChannel ch) {
+                    ch.pipeline().addLast(new ChannelHandlerAdapter() {});
+
+                }
+            }).group(group);
+            ChannelFuture connect = clientBootstrap.connect(remoteAddress.address());
+            connect.channel().closeFuture().addListener(future -> channelClosed.set(true));
+
+            assertBusy(() -> assertTrue("Channel should be closed due to read timeout", channelClosed.get()), 5, TimeUnit.SECONDS);
+
+        } finally {
+            group.shutdownGracefully().await();
+        }
+    }
 }

From 8b49b3f8afcd3539c890c0b0c149af12ce4daf24 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Fri, 8 Dec 2017 11:50:20 -0500
Subject: [PATCH 222/297] Remove unused import from AliasResolveRoutingIT

This commit removes an unused import from AliasResolveRoutingIT.java
that was left behind from development.
---
 .../java/org/elasticsearch/routing/AliasResolveRoutingIT.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java
index 7a0ce93e77d..dfebb3f754e 100644
--- a/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java
+++ b/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java
@@ -26,7 +26,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.test.ESIntegTestCase;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;

From 5c9415a4d3485729cfa65fdd71c911e8fa8f0127 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Fri, 8 Dec 2017 12:17:12 -0500
Subject: [PATCH 223/297] Cleanup split strings by comma method

We have some methods Strings#splitStringByCommaToArray and
Strings#splitStringByCommaToSet. It is not obvious that the former
leaves whitespace and the latter trims it. We also have
Strings#tokenizeToStringArray which tokenizes a string to an array, and
trims whitespace. It seems the right thing to do here is to rename
Strings#splitStringByCommaToSet to Strings#tokenizeByCommaToSet so that
its name is aligned with another method that tokenizes by a delimiter
and trims whitespace. We also cleanup the code here, removing an
unneeded splitting by delimiter to set method.

Relates #27715
---
 .../org/elasticsearch/common/Strings.java     | 117 ++++++------------
 .../rest/AbstractRestChannel.java             |   2 +-
 .../admin/cluster/RestNodesInfoAction.java    |   4 +-
 .../admin/cluster/RestNodesStatsAction.java   |   4 +-
 .../admin/cluster/RestNodesUsageAction.java   |   2 +-
 .../admin/indices/RestIndicesStatsAction.java |   2 +-
 .../elasticsearch/common/StringsTests.java    |  35 ++----
 7 files changed, 52 insertions(+), 114 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/Strings.java b/core/src/main/java/org/elasticsearch/common/Strings.java
index 13189a52b0b..6c2fc4e1ec1 100644
--- a/core/src/main/java/org/elasticsearch/common/Strings.java
+++ b/core/src/main/java/org/elasticsearch/common/Strings.java
@@ -41,6 +41,7 @@ import java.util.List;
 import java.util.Set;
 import java.util.StringTokenizer;
 import java.util.TreeSet;
+import java.util.function.Supplier;
 
 import static java.util.Collections.unmodifiableSet;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
@@ -410,62 +411,27 @@ public class Strings {
         return collection.toArray(new String[collection.size()]);
     }
 
-    public static Set splitStringByCommaToSet(final String s) {
-        return splitStringToSet(s, ',');
-    }
-
-    public static String[] splitStringByCommaToArray(final String s) {
-        if (s == null || s.isEmpty()) return Strings.EMPTY_ARRAY;
-        else return s.split(",");
+    /**
+     * Tokenize the specified string by commas to a set, trimming whitespace and ignoring empty tokens.
+     *
+     * @param s the string to tokenize
+     * @return the set of tokens
+     */
+    public static Set tokenizeByCommaToSet(final String s) {
+        if (s == null) return Collections.emptySet();
+        return tokenizeToCollection(s, ",", HashSet::new);
     }
 
     /**
-     * A convenience method for splitting a delimited string into
-     * a set and trimming leading and trailing whitespace from all
-     * split strings.
+     * Split the specified string by commas to an array.
      *
      * @param s the string to split
-     * @param c the delimiter to split on
-     * @return the set of split strings
+     * @return the array of split values
+     * @see String#split(String)
      */
-    public static Set splitStringToSet(final String s, final char c) {
-        if (s == null || s.isEmpty()) {
-            return Collections.emptySet();
-        }
-        final char[] chars = s.toCharArray();
-        int count = 1;
-        for (final char x : chars) {
-            if (x == c) {
-                count++;
-            }
-        }
-        final Set result = new HashSet<>(count);
-        final int len = chars.length;
-        int start = 0;  // starting index in chars of the current substring.
-        int pos = 0;    // current index in chars.
-        int end = 0; // the position of the end of the current token
-        for (; pos < len; pos++) {
-            if (chars[pos] == c) {
-                int size = end - start;
-                if (size > 0) { // only add non empty strings
-                    result.add(new String(chars, start, size));
-                }
-                start = pos + 1;
-                end = start;
-            } else if (Character.isWhitespace(chars[pos])) {
-                if (start == pos) {
-                    // skip over preceding whitespace
-                    start++;
-                }
-            } else {
-                end = pos + 1;
-            }
-        }
-        int size = end - start;
-        if (size > 0) {
-            result.add(new String(chars, start, size));
-        }
-        return result;
+    public static String[] splitStringByCommaToArray(final String s) {
+        if (s == null || s.isEmpty()) return Strings.EMPTY_ARRAY;
+        else return s.split(",");
     }
 
     /**
@@ -499,7 +465,7 @@ public class Strings {
      * tokens. A delimiter is always a single character; for multi-character
      * delimiters, consider using delimitedListToStringArray
      *
-     * @param str        the String to tokenize
+     * @param s        the String to tokenize
      * @param delimiters the delimiter characters, assembled as String
      *                   (each of those characters is individually considered as delimiter).
      * @return an array of the tokens
@@ -507,48 +473,35 @@ public class Strings {
      * @see java.lang.String#trim()
      * @see #delimitedListToStringArray
      */
-    public static String[] tokenizeToStringArray(String str, String delimiters) {
-        return tokenizeToStringArray(str, delimiters, true, true);
+    public static String[] tokenizeToStringArray(final String s, final String delimiters) {
+        return toStringArray(tokenizeToCollection(s, delimiters, ArrayList::new));
     }
 
     /**
-     * Tokenize the given String into a String array via a StringTokenizer.
-     * 
    The given delimiters string is supposed to consist of any number of
-     * delimiter characters. Each of those characters can be used to separate
-     * tokens. A delimiter is always a single character; for multi-character
-     * delimiters, consider using delimitedListToStringArray
+     * Tokenizes the specified string to a collection using the specified delimiters as the token delimiters. This method trims whitespace
+     * from tokens and ignores empty tokens.
      *
-     * @param str               the String to tokenize
-     * @param delimiters        the delimiter characters, assembled as String
-     *                          (each of those characters is individually considered as delimiter)
-     * @param trimTokens        trim the tokens via String's trim
-     * @param ignoreEmptyTokens omit empty tokens from the result array
-     *                          (only applies to tokens that are empty after trimming; StringTokenizer
-     *                          will not consider subsequent delimiters as token in the first place).
-     * @return an array of the tokens (null if the input String
-     *         was null)
+     * @param s          the string to tokenize.
+     * @param delimiters the token delimiters
+     * @param supplier   a collection supplier
+     * @param         the type of the collection
+     * @return the tokens
      * @see java.util.StringTokenizer
-     * @see java.lang.String#trim()
-     * @see #delimitedListToStringArray
      */
-    public static String[] tokenizeToStringArray(
-            String str, String delimiters, boolean trimTokens, boolean ignoreEmptyTokens) {
-
-        if (str == null) {
+    private static > T tokenizeToCollection(
+            final String s, final String delimiters, final Supplier supplier) {
+        if (s == null) {
             return null;
         }
-        StringTokenizer st = new StringTokenizer(str, delimiters);
-        List tokens = new ArrayList<>();
-        while (st.hasMoreTokens()) {
-            String token = st.nextToken();
-            if (trimTokens) {
-                token = token.trim();
-            }
-            if (!ignoreEmptyTokens || token.length() > 0) {
+        final StringTokenizer tokenizer = new StringTokenizer(s, delimiters);
+        final T tokens = supplier.get();
+        while (tokenizer.hasMoreTokens()) {
+            final String token = tokenizer.nextToken().trim();
+            if (token.length() > 0) {
                 tokens.add(token);
             }
         }
-        return toStringArray(tokens);
+        return tokens;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
index 4db9aec6e93..6c84c1bb963 100644
--- a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
+++ b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
@@ -94,7 +94,7 @@ public abstract class AbstractRestChannel implements RestChannel {
         Set includes = Collections.emptySet();
         Set excludes = Collections.emptySet();
         if (useFiltering) {
-            Set filters = Strings.splitStringByCommaToSet(filterPath);
+            Set filters = Strings.tokenizeByCommaToSet(filterPath);
             includes = filters.stream().filter(INCLUDE_FILTER).collect(toSet());
             excludes = filters.stream().filter(EXCLUDE_FILTER).map(f -> f.substring(1)).collect(toSet());
         }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
index 256693fc392..bacc698b2a4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
@@ -76,7 +76,7 @@ public class RestNodesInfoAction extends BaseRestHandler {
         // still, /_nodes/_local (or any other node id) should work and be treated as usual
         // this means one must differentiate between allowed metrics and arbitrary node ids in the same place
         if (request.hasParam("nodeId") && !request.hasParam("metrics")) {
-            Set metricsOrNodeIds = Strings.splitStringByCommaToSet(request.param("nodeId", "_all"));
+            Set metricsOrNodeIds = Strings.tokenizeByCommaToSet(request.param("nodeId", "_all"));
             boolean isMetricsOnly = ALLOWED_METRICS.containsAll(metricsOrNodeIds);
             if (isMetricsOnly) {
                 nodeIds = new String[]{"_all"};
@@ -87,7 +87,7 @@ public class RestNodesInfoAction extends BaseRestHandler {
             }
         } else {
             nodeIds = Strings.splitStringByCommaToArray(request.param("nodeId", "_all"));
-            metrics = Strings.splitStringByCommaToSet(request.param("metrics", "_all"));
+            metrics = Strings.tokenizeByCommaToSet(request.param("metrics", "_all"));
         }
 
         final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(nodeIds);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
index 28f8163760f..4e1b158a18c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
@@ -92,7 +92,7 @@ public class RestNodesStatsAction extends BaseRestHandler {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
-        Set metrics = Strings.splitStringByCommaToSet(request.param("metric", "_all"));
+        Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all"));
 
         NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(nodesIds);
         nodesStatsRequest.timeout(request.param("timeout"));
@@ -134,7 +134,7 @@ public class RestNodesStatsAction extends BaseRestHandler {
 
             // check for index specific metrics
             if (metrics.contains("indices")) {
-                Set indexMetrics = Strings.splitStringByCommaToSet(request.param("index_metric", "_all"));
+                Set indexMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", "_all"));
                 if (indexMetrics.size() == 1 && indexMetrics.contains("_all")) {
                     nodesStatsRequest.indices(CommonStatsFlags.ALL);
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java
index b22f63ca78d..dc6449862a8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesUsageAction.java
@@ -56,7 +56,7 @@ public class RestNodesUsageAction extends BaseRestHandler {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
-        Set metrics = Strings.splitStringByCommaToSet(request.param("metric", "_all"));
+        Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all"));
 
         NodesUsageRequest nodesUsageRequest = new NodesUsageRequest(nodesIds);
         nodesUsageRequest.timeout(request.param("timeout"));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
index 5458a60e141..ca554301b93 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
@@ -91,7 +91,7 @@ public class RestIndicesStatsAction extends BaseRestHandler {
         indicesStatsRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
         indicesStatsRequest.types(Strings.splitStringByCommaToArray(request.param("types")));
 
-        Set metrics = Strings.splitStringByCommaToSet(request.param("metric", "_all"));
+        Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all"));
         // short cut, if no metrics have been specified in URI
         if (metrics.size() == 1 && metrics.contains("_all")) {
             indicesStatsRequest.all();
diff --git a/core/src/test/java/org/elasticsearch/common/StringsTests.java b/core/src/test/java/org/elasticsearch/common/StringsTests.java
index 2ca5528addd..af065408a99 100644
--- a/core/src/test/java/org/elasticsearch/common/StringsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/StringsTests.java
@@ -90,30 +90,15 @@ public class StringsTests extends ESTestCase {
     }
 
     public void testSplitStringToSet() {
-        assertEquals(Strings.splitStringByCommaToSet(null), Sets.newHashSet());
-        assertEquals(Strings.splitStringByCommaToSet(""), Sets.newHashSet());
-        assertEquals(Strings.splitStringByCommaToSet("a,b,c"), Sets.newHashSet("a","b","c"));
-        assertEquals(Strings.splitStringByCommaToSet("a, b, c"), Sets.newHashSet("a","b","c"));
-        assertEquals(Strings.splitStringByCommaToSet(" a ,  b, c  "), Sets.newHashSet("a","b","c"));
-        assertEquals(Strings.splitStringByCommaToSet("aa, bb, cc"), Sets.newHashSet("aa","bb","cc"));
-        assertEquals(Strings.splitStringByCommaToSet(" a "), Sets.newHashSet("a"));
-        assertEquals(Strings.splitStringByCommaToSet("   a   "), Sets.newHashSet("a"));
-        assertEquals(Strings.splitStringByCommaToSet("   aa   "), Sets.newHashSet("aa"));
-        assertEquals(Strings.splitStringByCommaToSet("   "), Sets.newHashSet());
-
-        assertEquals(Strings.splitStringToSet(null, ' '), Sets.newHashSet());
-        assertEquals(Strings.splitStringToSet("", ' '), Sets.newHashSet());
-        assertEquals(Strings.splitStringToSet("a b c", ' '), Sets.newHashSet("a","b","c"));
-        assertEquals(Strings.splitStringToSet("a, b, c", ' '), Sets.newHashSet("a,","b,","c"));
-        assertEquals(Strings.splitStringToSet(" a   b c  ", ' '), Sets.newHashSet("a","b","c"));
-        assertEquals(Strings.splitStringToSet("  a   b   c  ", ' '), Sets.newHashSet("a","b","c"));
-        assertEquals(Strings.splitStringToSet("aa bb cc", ' '), Sets.newHashSet("aa","bb","cc"));
-        assertEquals(Strings.splitStringToSet(" a ", ' '), Sets.newHashSet("a"));
-        assertEquals(Strings.splitStringToSet("    a    ", ' '), Sets.newHashSet("a"));
-        assertEquals(Strings.splitStringToSet(" a   ", ' '), Sets.newHashSet("a"));
-        assertEquals(Strings.splitStringToSet("a   ", ' '), Sets.newHashSet("a"));
-        assertEquals(Strings.splitStringToSet("   aa   ", ' '), Sets.newHashSet("aa"));
-        assertEquals(Strings.splitStringToSet("aa   ", ' '), Sets.newHashSet("aa"));
-        assertEquals(Strings.splitStringToSet("   ", ' '), Sets.newHashSet());
+        assertEquals(Strings.tokenizeByCommaToSet(null), Sets.newHashSet());
+        assertEquals(Strings.tokenizeByCommaToSet(""), Sets.newHashSet());
+        assertEquals(Strings.tokenizeByCommaToSet("a,b,c"), Sets.newHashSet("a","b","c"));
+        assertEquals(Strings.tokenizeByCommaToSet("a, b, c"), Sets.newHashSet("a","b","c"));
+        assertEquals(Strings.tokenizeByCommaToSet(" a ,  b, c  "), Sets.newHashSet("a","b","c"));
+        assertEquals(Strings.tokenizeByCommaToSet("aa, bb, cc"), Sets.newHashSet("aa","bb","cc"));
+        assertEquals(Strings.tokenizeByCommaToSet(" a "), Sets.newHashSet("a"));
+        assertEquals(Strings.tokenizeByCommaToSet("   a   "), Sets.newHashSet("a"));
+        assertEquals(Strings.tokenizeByCommaToSet("   aa   "), Sets.newHashSet("aa"));
+        assertEquals(Strings.tokenizeByCommaToSet("   "), Sets.newHashSet());
     }
 }

From d21167e0c2698422b3cc5ad9ad74e9f36796382e Mon Sep 17 00:00:00 2001
From: David Roberts 
Date: Fri, 8 Dec 2017 17:37:05 +0000
Subject: [PATCH 224/297] [TEST] Remove leftover ES temp directories before
 Vagrant tests (#27722)

Some of the Vagrant tests were failing due to ES temp directories
left over from previous uses of the same VM confusing subsequent
tests into thinking there were multiple ES installs present.

This change wipes all ES temp directories when the test VMs are
brought up.
---
 Vagrantfile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Vagrantfile b/Vagrantfile
index 021b4d630a1..6ace825e5c3 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -307,4 +307,9 @@ Defaults   env_keep += "BATS_ARCHIVES"
 SUDOERS_VARS
     chmod 0440 /etc/sudoers.d/elasticsearch_vars
   SHELL
+  # This prevents leftovers from previous tests using the
+  # same VM from messing up the current test
+  config.vm.provision "clean_tmp", run: "always", type: "shell", inline: <<-SHELL
+    rm -rf /tmp/elasticsearch*
+  SHELL
 end

From d82c40d35c07362840ca942a1d93fe3b5ad121cf Mon Sep 17 00:00:00 2001
From: Tim Brooks 
Date: Fri, 8 Dec 2017 10:39:30 -0700
Subject: [PATCH 225/297] Implement byte array reusage in `NioTransport`
 (#27696)

This is related to #27563. This commit modifies the
InboundChannelBuffer to support releasable byte pages. These byte
pages are provided by the PageCacheRecycler. The PageCacheRecycler
must be passed to the Transport with this change.
---
 .../client/transport/TransportClient.java     |  6 +-
 .../common/network/NetworkModule.java         |  6 +-
 .../elasticsearch/common/util/BigArrays.java  |  4 +-
 .../common/util/PageCacheRecycler.java        |  6 +-
 .../java/org/elasticsearch/node/Node.java     | 19 ++++-
 .../elasticsearch/plugins/NetworkPlugin.java  |  2 +
 .../ReleasableBytesStreamOutputTests.java     |  3 +-
 .../common/network/NetworkModuleTests.java    |  8 +-
 .../common/util/BigArraysTests.java           |  2 +-
 .../common/util/BytesRefHashTests.java        |  2 +-
 .../common/util/LongHashTests.java            |  2 +-
 .../common/util/LongObjectHashMapTests.java   |  2 +-
 .../elasticsearch/index/IndexModuleTests.java |  4 +-
 .../search/DefaultSearchContextTests.java     |  3 +-
 .../MultiBucketAggregatorWrapperTests.java    |  3 +-
 .../BestDocsDeferringCollectorTests.java      |  5 +-
 .../bucket/terms/TermsAggregatorTests.java    |  3 +-
 .../cardinality/InternalCardinalityTests.java |  5 +-
 .../transport/TcpTransportTests.java          |  3 +-
 .../stats/InternalMatrixStatsTests.java       |  3 +-
 .../elasticsearch/transport/Netty4Plugin.java |  2 +
 .../http/netty4/Netty4HttpChannelTests.java   |  3 +-
 .../Netty4HttpServerPipeliningTests.java      |  3 +-
 .../Netty4HttpServerTransportTests.java       |  3 +-
 .../Netty4SizeHeaderFrameDecoderTests.java    |  3 +-
 .../transport/netty4/Netty4TransportIT.java   |  2 +
 .../netty4/NettyTransportMultiPortTests.java  |  3 +-
 .../common/util/MockBigArrays.java            |  4 +-
 .../common/util/MockPageCacheRecycler.java    |  2 +-
 .../java/org/elasticsearch/node/MockNode.java | 16 +++-
 .../aggregations/AggregatorTestCase.java      |  3 +-
 .../elasticsearch/test/ESIntegTestCase.java   |  2 +-
 .../org/elasticsearch/test/ESTestCase.java    |  8 +-
 .../test/InternalAggregationTestCase.java     |  3 +-
 .../transport/MockTcpTransportPlugin.java     |  2 +
 .../transport/nio/InboundChannelBuffer.java   | 75 ++++++++++++++-----
 .../transport/nio/NioTransport.java           | 18 ++++-
 .../transport/nio/NioTransportPlugin.java     |  5 +-
 .../nio/channel/NioSocketChannel.java         |  1 +
 .../transport/nio/channel/ReadContext.java    |  5 +-
 .../transport/nio/channel/TcpReadContext.java | 20 ++---
 .../nio/InboundChannelBufferTests.java        | 74 ++++++++++++++++--
 .../nio/SimpleNioTransportTests.java          |  6 +-
 .../nio/channel/TcpReadContextTests.java      | 17 ++++-
 44 files changed, 280 insertions(+), 91 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index 182c2532214..0d4a87f3bfa 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -43,6 +43,7 @@ import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.node.InternalSettingsPreparer;
@@ -169,11 +170,12 @@ public abstract class TransportClient extends AbstractClient {
             CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(),
                 settingsModule.getClusterSettings());
             resourcesToClose.add(circuitBreakerService);
-            BigArrays bigArrays = new BigArrays(settings, circuitBreakerService);
+            PageCacheRecycler pageCacheRecycler = new PageCacheRecycler(settings);
+            BigArrays bigArrays = new BigArrays(pageCacheRecycler, circuitBreakerService);
             resourcesToClose.add(bigArrays);
             modules.add(settingsModule);
             NetworkModule networkModule = new NetworkModule(settings, true, pluginsService.filterPlugins(NetworkPlugin.class), threadPool,
-                bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, null);
+                bigArrays, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, null);
             final Transport transport = networkModule.getTransportSupplier().get();
             final TransportService transportService = new TransportService(settings, transport, threadPool,
                 networkModule.getTransportInterceptor(),
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index 8cb13647fb6..2b9cde65d64 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.http.HttpServerTransport;
@@ -107,6 +108,7 @@ public final class NetworkModule {
      */
     public NetworkModule(Settings settings, boolean transportClient, List plugins, ThreadPool threadPool,
                          BigArrays bigArrays,
+                         PageCacheRecycler pageCacheRecycler,
                          CircuitBreakerService circuitBreakerService,
                          NamedWriteableRegistry namedWriteableRegistry,
                          NamedXContentRegistry xContentRegistry,
@@ -121,9 +123,9 @@ public final class NetworkModule {
                     registerHttpTransport(entry.getKey(), entry.getValue());
                 }
             }
-            Map> httpTransportFactory = plugin.getTransports(settings, threadPool, bigArrays,
+            Map> transportFactory = plugin.getTransports(settings, threadPool, bigArrays, pageCacheRecycler,
                 circuitBreakerService, namedWriteableRegistry, networkService);
-            for (Map.Entry> entry : httpTransportFactory.entrySet()) {
+            for (Map.Entry> entry : transportFactory.entrySet()) {
                 registerTransport(entry.getKey(), entry.getValue());
             }
             List transportInterceptors = plugin.getTransportInterceptors(namedWriteableRegistry,
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
index 5c539a791cf..ec46a2b937f 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
@@ -372,9 +372,9 @@ public class BigArrays implements Releasable {
     final boolean checkBreaker;
     private final BigArrays circuitBreakingInstance;
 
-    public BigArrays(Settings settings, @Nullable final CircuitBreakerService breakerService) {
+    public BigArrays(PageCacheRecycler recycler, @Nullable final CircuitBreakerService breakerService) {
         // Checking the breaker is disabled if not specified
-        this(new PageCacheRecycler(settings), breakerService, false);
+        this(recycler, breakerService, false);
     }
 
     // public for tests
diff --git a/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java b/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
index f40938b8ec0..be69cf5b95a 100644
--- a/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
+++ b/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
@@ -65,10 +65,10 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable {
         Releasables.close(true, bytePage, intPage, longPage, objectPage);
     }
 
-    protected PageCacheRecycler(Settings settings) {
+    public PageCacheRecycler(Settings settings) {
         super(settings);
-        final Type type = TYPE_SETTING .get(settings);
-        final long limit = LIMIT_HEAP_SETTING .get(settings).getBytes();
+        final Type type = TYPE_SETTING.get(settings);
+        final long limit = LIMIT_HEAP_SETTING.get(settings).getBytes();
         final int availableProcessors = EsExecutors.numberOfProcessors(settings);
 
         // We have a global amount of memory that we need to divide across data types.
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index a4b7e5147d5..2a677d00afe 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -80,6 +80,7 @@ import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -363,7 +364,8 @@ public class Node implements Closeable {
             modules.add(new GatewayModule());
 
 
-            BigArrays bigArrays = createBigArrays(settings, circuitBreakerService);
+            PageCacheRecycler pageCacheRecycler = createPageCacheRecycler(settings);
+            BigArrays bigArrays = createBigArrays(pageCacheRecycler, circuitBreakerService);
             resourcesToClose.add(bigArrays);
             modules.add(settingsModule);
             List namedWriteables = Stream.of(
@@ -403,7 +405,8 @@ public class Node implements Closeable {
 
             final RestController restController = actionModule.getRestController();
             final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class),
-                    threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, restController);
+                threadPool, bigArrays, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, xContentRegistry,
+                networkService, restController);
             Collection>> customMetaDataUpgraders =
                 pluginsService.filterPlugins(Plugin.class).stream()
                     .map(Plugin::getCustomMetaDataUpgrader)
@@ -898,8 +901,16 @@ public class Node implements Closeable {
      * Creates a new {@link BigArrays} instance used for this node.
      * This method can be overwritten by subclasses to change their {@link BigArrays} implementation for instance for testing
      */
-    BigArrays createBigArrays(Settings settings, CircuitBreakerService circuitBreakerService) {
-        return new BigArrays(settings, circuitBreakerService);
+    BigArrays createBigArrays(PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService) {
+        return new BigArrays(pageCacheRecycler, circuitBreakerService);
+    }
+
+    /**
+     * Creates a new {@link BigArrays} instance used for this node.
+     * This method can be overwritten by subclasses to change their {@link BigArrays} implementation for instance for testing
+     */
+    PageCacheRecycler createPageCacheRecycler(Settings settings) {
+        return new PageCacheRecycler(settings);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/plugins/NetworkPlugin.java b/core/src/main/java/org/elasticsearch/plugins/NetworkPlugin.java
index 33fab61c24a..df41036ffea 100644
--- a/core/src/main/java/org/elasticsearch/plugins/NetworkPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/NetworkPlugin.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.http.HttpServerTransport;
@@ -58,6 +59,7 @@ public interface NetworkPlugin {
      * See {@link org.elasticsearch.common.network.NetworkModule#TRANSPORT_TYPE_KEY} to configure a specific implementation.
      */
     default Map> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                           PageCacheRecycler pageCacheRecycler,
                                                            CircuitBreakerService circuitBreakerService,
                                                            NamedWriteableRegistry namedWriteableRegistry,
                                                            NetworkService networkService) {
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutputTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutputTests.java
index 557721a0241..5a85310adcd 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutputTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/ReleasableBytesStreamOutputTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.common.io.stream;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 
@@ -30,7 +31,7 @@ public class ReleasableBytesStreamOutputTests extends ESTestCase {
 
     public void testRelease() throws Exception {
         MockBigArrays mockBigArrays =
-            new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+            new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         try (ReleasableBytesStreamOutput output =
                  getRandomReleasableBytesStreamOutput(mockBigArrays)) {
             output.writeBoolean(randomBoolean());
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index 81e685e6852..70deb8a4ba8 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.http.HttpInfo;
@@ -133,6 +134,7 @@ public class NetworkModuleTests extends ModuleTestCase {
         NetworkPlugin plugin = new NetworkPlugin() {
             @Override
             public Map> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                  PageCacheRecycler pageCacheRecycler,
                                                                   CircuitBreakerService circuitBreakerService,
                                                                   NamedWriteableRegistry namedWriteableRegistry,
                                                                   NetworkService networkService) {
@@ -193,6 +195,7 @@ public class NetworkModuleTests extends ModuleTestCase {
         NetworkModule module = newNetworkModule(settings, false, new NetworkPlugin() {
             @Override
             public Map> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                  PageCacheRecycler pageCacheRecycler,
                                                                   CircuitBreakerService circuitBreakerService,
                                                                   NamedWriteableRegistry namedWriteableRegistry,
                                                                   NetworkService networkService) {
@@ -227,6 +230,7 @@ public class NetworkModuleTests extends ModuleTestCase {
         NetworkModule module = newNetworkModule(settings, false, new NetworkPlugin() {
             @Override
             public Map> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                  PageCacheRecycler pageCacheRecycler,
                                                                   CircuitBreakerService circuitBreakerService,
                                                                   NamedWriteableRegistry namedWriteableRegistry,
                                                                   NetworkService networkService) {
@@ -306,7 +310,7 @@ public class NetworkModuleTests extends ModuleTestCase {
     }
 
     private NetworkModule newNetworkModule(Settings settings, boolean transportClient, NetworkPlugin... plugins) {
-        return new NetworkModule(settings, transportClient, Arrays.asList(plugins), threadPool, null, null, null, xContentRegistry(), null,
-            new NullDispatcher());
+        return new NetworkModule(settings, transportClient, Arrays.asList(plugins), threadPool, null, null, null, null,
+            xContentRegistry(), null, new NullDispatcher());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java b/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java
index 945dda446ce..f280b9a80bf 100644
--- a/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java
@@ -42,7 +42,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 public class BigArraysTests extends ESTestCase {
 
     private BigArrays randombigArrays() {
-        return new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
     }
 
     private BigArrays bigArrays;
diff --git a/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java b/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
index 78abed0b320..5370815926c 100644
--- a/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
@@ -41,7 +41,7 @@ public class BytesRefHashTests extends ESSingleNodeTestCase {
     BytesRefHash hash;
 
     private BigArrays randombigArrays() {
-        return new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
     }
 
     private void newHash() {
diff --git a/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java b/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java
index 66c9e689c96..708e7beb861 100644
--- a/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java
@@ -36,7 +36,7 @@ public class LongHashTests extends ESSingleNodeTestCase {
     LongHash hash;
 
     private BigArrays randombigArrays() {
-        return new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
     }
 
     private void newHash() {
diff --git a/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java b/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java
index f783317808d..9210565a104 100644
--- a/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java
@@ -27,7 +27,7 @@ import org.elasticsearch.test.ESSingleNodeTestCase;
 public class LongObjectHashMapTests extends ESSingleNodeTestCase {
 
     private BigArrays randombigArrays() {
-        return new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
     }
 
     public void testDuel() {
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index f1037d67ff4..c7d396c778a 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -39,6 +39,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLock;
@@ -124,7 +125,8 @@ public class IndexModuleTests extends ESTestCase {
                 emptyMap(), emptyMap(), emptyMap());
         threadPool = new TestThreadPool("test");
         circuitBreakerService = new NoneCircuitBreakerService();
-        bigArrays = new BigArrays(settings, circuitBreakerService);
+        PageCacheRecycler pageCacheRecycler = new PageCacheRecycler(settings);
+        bigArrays = new BigArrays(pageCacheRecycler, circuitBreakerService);
         scriptService = new ScriptService(settings, Collections.emptyMap(), Collections.emptyMap());
         clusterService = ClusterServiceUtils.createClusterService(threadPool);
         nodeEnvironment = new NodeEnvironment(settings, environment);
diff --git a/core/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/core/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 539bdef17d3..ec422435e4e 100644
--- a/core/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/core/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.cache.IndexCache;
@@ -104,7 +105,7 @@ public class DefaultSearchContextTests extends ESTestCase {
         when(indexService.getIndexSettings()).thenReturn(indexSettings);
         when(mapperService.getIndexSettings()).thenReturn(indexSettings);
 
-        BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
 
         try (Directory dir = newDirectory();
              RandomIndexWriter w = new RandomIndexWriter(random(), dir);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/MultiBucketAggregatorWrapperTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/MultiBucketAggregatorWrapperTests.java
index 0a83b2ec5c7..9c2ae31af14 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/MultiBucketAggregatorWrapperTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/MultiBucketAggregatorWrapperTests.java
@@ -25,6 +25,7 @@ import org.apache.lucene.search.Scorer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
@@ -46,7 +47,7 @@ public class MultiBucketAggregatorWrapperTests extends ESTestCase {
     public void testNoNullScorerIsDelegated() throws Exception {
         LeafReaderContext leafReaderContext = MemoryIndex.fromDocument(Collections.emptyList(), new MockAnalyzer(random()))
                 .createSearcher().getIndexReader().leaves().get(0);
-        BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         SearchContext searchContext = mock(SearchContext.class);
         when(searchContext.bigArrays()).thenReturn(bigArrays);
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollectorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollectorTests.java
index d99f7e9fa73..86e937a356b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollectorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/BestDocsDeferringCollectorTests.java
@@ -33,6 +33,7 @@ import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.BucketCollector;
@@ -63,8 +64,8 @@ public class BestDocsDeferringCollectorTests extends AggregatorTestCase {
         TermQuery termQuery = new TermQuery(new Term("field", String.valueOf(randomInt(maxNumValues))));
         TopDocs topDocs = indexSearcher.search(termQuery, numDocs);
 
-        BestDocsDeferringCollector collector =
-                new BestDocsDeferringCollector(numDocs, new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()));
+        BestDocsDeferringCollector collector = new BestDocsDeferringCollector(numDocs,
+            new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()));
         Set deferredCollectedDocIds = new HashSet<>();
         collector.setDeferredCollector(Collections.singleton(testCollector(deferredCollectedDocIds)));
         collector.preCollection();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
index 9dd355d8ca0..74cc35da300 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
@@ -38,6 +38,7 @@ import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.IpFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
@@ -912,7 +913,7 @@ public class TermsAggregatorTests extends AggregatorTestCase {
                 dir.close();
             }
             InternalAggregation.ReduceContext ctx =
-                new InternalAggregation.ReduceContext(new MockBigArrays(Settings.EMPTY,
+                new InternalAggregation.ReduceContext(new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY),
                     new NoneCircuitBreakerService()), null, true);
             for (InternalAggregation internalAgg : aggs) {
                 InternalAggregation mergedAggs = internalAgg.doReduce(aggs, ctx);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java
index 66e9644cb42..fc1095c857f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinalityTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.aggregations.ParsedAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -60,7 +61,7 @@ public class InternalCardinalityTests extends InternalAggregationTestCase pipelineAggregators, Map metaData) {
         HyperLogLogPlusPlus hllpp = new HyperLogLogPlusPlus(p,
-                new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()), 1);
+                new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()), 1);
         algos.add(hllpp);
         for (int i = 0; i < 100; i++) {
             hllpp.collect(0, BitMixer.mix64(randomIntBetween(1, 100)));
@@ -107,7 +108,7 @@ public class InternalCardinalityTests extends InternalAggregationTestCase> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                          PageCacheRecycler pageCacheRecycler,
                                                           CircuitBreakerService circuitBreakerService,
                                                           NamedWriteableRegistry namedWriteableRegistry,
                                                           NetworkService networkService) {
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java
index 4b086ab4650..e9de4ef50a5 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java
@@ -52,6 +52,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ByteArray;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -99,7 +100,7 @@ public class Netty4HttpChannelTests extends ESTestCase {
     public void setup() throws Exception {
         networkService = new NetworkService(Collections.emptyList());
         threadPool = new TestThreadPool("test");
-        bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
     }
 
     @After
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
index 4fdd842cb19..91a5465f6a7 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
@@ -36,6 +36,7 @@ import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.NullDispatcher;
@@ -74,7 +75,7 @@ public class Netty4HttpServerPipeliningTests extends ESTestCase {
     public void setup() throws Exception {
         networkService = new NetworkService(Collections.emptyList());
         threadPool = new TestThreadPool("test");
-        bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
     }
 
     @After
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
index c347bd2805d..96b436ce7de 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
@@ -48,6 +48,7 @@ import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.BindHttpException;
 import org.elasticsearch.http.HttpServerTransport;
@@ -103,7 +104,7 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
     public void setup() throws Exception {
         networkService = new NetworkService(Collections.emptyList());
         threadPool = new TestThreadPool("test");
-        bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
     }
 
     @After
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
index 7c56fdc3ab4..7343da6c3b1 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.mocksocket.MockSocket;
 import org.elasticsearch.test.ESTestCase;
@@ -63,7 +64,7 @@ public class Netty4SizeHeaderFrameDecoderTests extends ESTestCase {
     public void startThreadPool() {
         threadPool = new ThreadPool(settings);
         NetworkService networkService = new NetworkService(Collections.emptyList());
-        BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         nettyTransport = new Netty4Transport(settings, threadPool, networkService, bigArrays,
             new NamedWriteableRegistry(Collections.emptyList()), new NoneCircuitBreakerService());
         nettyTransport.start();
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java
index 04a2b8131f9..b81c8efcb47 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -90,6 +91,7 @@ public class Netty4TransportIT extends ESNetty4IntegTestCase {
 
             @Override
             public Map> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                                  PageCacheRecycler pageCacheRecycler,
                                                                   CircuitBreakerService circuitBreakerService,
                                                                   NamedWriteableRegistry namedWriteableRegistry,
                                                                   NetworkService networkService) {
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java
index cde939bab8d..a49df3caaba 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -116,7 +117,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
     }
 
     private TcpTransport startTransport(Settings settings, ThreadPool threadPool) {
-        BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         TcpTransport transport = new Netty4Transport(settings, threadPool, new NetworkService(Collections.emptyList()),
             bigArrays, new NamedWriteableRegistry(Collections.emptyList()), new NoneCircuitBreakerService());
         transport.start();
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
index 2f75d92d898..9eeca0bd12d 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
@@ -70,8 +70,8 @@ public class MockBigArrays extends BigArrays {
     private final PageCacheRecycler recycler;
     private final CircuitBreakerService breakerService;
 
-    public MockBigArrays(Settings settings, CircuitBreakerService breakerService) {
-        this(new MockPageCacheRecycler(settings), breakerService, false);
+    public MockBigArrays(PageCacheRecycler recycler, CircuitBreakerService breakerService) {
+        this(recycler, breakerService, false);
     }
 
     private MockBigArrays(PageCacheRecycler recycler, CircuitBreakerService breakerService, boolean checkBreaker) {
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockPageCacheRecycler.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockPageCacheRecycler.java
index 766d68f45dd..5fcf2f11d0e 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/util/MockPageCacheRecycler.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockPageCacheRecycler.java
@@ -57,7 +57,7 @@ public class MockPageCacheRecycler extends PageCacheRecycler {
 
     private final Random random;
 
-    MockPageCacheRecycler(Settings settings) {
+    public MockPageCacheRecycler(Settings settings) {
         super(settings);
         // we always initialize with 0 here since we really only wanna have some random bytes / ints / longs
         // and given the fact that it's called concurrently it won't reproduces anyway the same order other than in a unittest
diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
index 34ed8d337e8..5a6075b1f85 100644
--- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
+++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
@@ -30,6 +30,8 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -82,11 +84,19 @@ public class MockNode extends Node {
     }
 
     @Override
-    protected BigArrays createBigArrays(Settings settings, CircuitBreakerService circuitBreakerService) {
+    protected BigArrays createBigArrays(PageCacheRecycler pageCacheRecycler, CircuitBreakerService circuitBreakerService) {
         if (getPluginsService().filterPlugins(NodeMocksPlugin.class).isEmpty()) {
-            return super.createBigArrays(settings, circuitBreakerService);
+            return super.createBigArrays(pageCacheRecycler, circuitBreakerService);
         }
-        return new MockBigArrays(settings, circuitBreakerService);
+        return new MockBigArrays(pageCacheRecycler, circuitBreakerService);
+    }
+
+    @Override
+    PageCacheRecycler createPageCacheRecycler(Settings settings) {
+        if (getPluginsService().filterPlugins(NodeMocksPlugin.class).isEmpty()) {
+            return super.createPageCacheRecycler(settings);
+        }
+        return new MockPageCacheRecycler(settings);
     }
 
 
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index 8ae6c7843d2..f34b1c6e79f 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -38,6 +38,7 @@ import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
@@ -112,7 +113,7 @@ public abstract class AggregatorTestCase extends ESTestCase {
         CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
         when(searchContext.aggregations())
             .thenReturn(new SearchContextAggregations(AggregatorFactories.EMPTY, bucketConsumer));
-        when(searchContext.bigArrays()).thenReturn(new MockBigArrays(Settings.EMPTY, circuitBreakerService));
+        when(searchContext.bigArrays()).thenReturn(new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), circuitBreakerService));
         // TODO: now just needed for top_hits, this will need to be revised for other agg unit tests:
         MapperService mapperService = mapperServiceMock();
         when(mapperService.getIndexSettings()).thenReturn(indexSettings);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 1dbf4015546..be6a1b29681 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -2076,7 +2076,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
             try {
                 INSTANCE.printTestMessage("cleaning up after");
                 INSTANCE.afterInternal(true);
-                checkStaticState();
+                checkStaticState(true);
             } finally {
                 INSTANCE = null;
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index e10411e5a43..01ad6763a8c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -288,7 +288,7 @@ public abstract class ESTestCase extends LuceneTestCase {
 
     @After
     public final void after() throws Exception {
-        checkStaticState();
+        checkStaticState(false);
         // We check threadContext != null rather than enableWarningsCheck()
         // because after methods are still called in the event that before
         // methods failed, in which case threadContext might not have been
@@ -394,8 +394,10 @@ public abstract class ESTestCase extends LuceneTestCase {
     }
 
     // separate method so that this can be checked again after suite scoped cluster is shut down
-    protected static void checkStaticState() throws Exception {
-        MockPageCacheRecycler.ensureAllPagesAreReleased();
+    protected static void checkStaticState(boolean afterClass) throws Exception {
+        if (afterClass) {
+            MockPageCacheRecycler.ensureAllPagesAreReleased();
+        }
         MockBigArrays.ensureAllArraysAreReleased();
 
         // ensure no one changed the status logger level on us
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
index 9de83aee630..ea846c5dd18 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.xcontent.ContextParser;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -236,7 +237,7 @@ public abstract class InternalAggregationTestCase
             toReduce.add(t);
         }
         ScriptService mockScriptService = mockScriptService();
-        MockBigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
+        MockBigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         if (randomBoolean() && toReduce.size() > 1) {
             // sometimes do an incremental reduce
             Collections.shuffle(toReduce, random());
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransportPlugin.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransportPlugin.java
index 25c356ebf4d..c85fee985fe 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransportPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransportPlugin.java
@@ -23,6 +23,7 @@ import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -39,6 +40,7 @@ public class MockTcpTransportPlugin extends Plugin implements NetworkPlugin {
 
     @Override
     public Map> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                          PageCacheRecycler pageCacheRecycler,
                                                           CircuitBreakerService circuitBreakerService,
                                                           NamedWriteableRegistry namedWriteableRegistry,
                                                           NetworkService networkService) {
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/InboundChannelBuffer.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/InboundChannelBuffer.java
index 46cec52bb6c..11d340a30a0 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/InboundChannelBuffer.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/InboundChannelBuffer.java
@@ -19,9 +19,16 @@
 
 package org.elasticsearch.transport.nio;
 
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.util.BigArrays;
+
 import java.nio.ByteBuffer;
 import java.util.ArrayDeque;
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
 
 /**
@@ -30,39 +37,56 @@ import java.util.function.Supplier;
  * the pages internally. If more space is needed at the end of the buffer {@link #ensureCapacity(long)} can
  * be called and the buffer will expand using the supplier provided.
  */
-public final class InboundChannelBuffer {
+public final class InboundChannelBuffer implements Releasable {
 
-    private static final int PAGE_SIZE = 1 << 14;
+    private static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE;
     private static final int PAGE_MASK = PAGE_SIZE - 1;
     private static final int PAGE_SHIFT = Integer.numberOfTrailingZeros(PAGE_SIZE);
     private static final ByteBuffer[] EMPTY_BYTE_BUFFER_ARRAY = new ByteBuffer[0];
 
 
-    private final ArrayDeque pages;
-    private final Supplier pageSupplier;
+    private final ArrayDeque pages;
+    private final Supplier pageSupplier;
+    private final AtomicBoolean isClosed = new AtomicBoolean(false);
 
     private long capacity = 0;
     private long internalIndex = 0;
     // The offset is an int as it is the offset of where the bytes begin in the first buffer
     private int offset = 0;
 
-    public InboundChannelBuffer() {
-        this(() -> ByteBuffer.wrap(new byte[PAGE_SIZE]));
-    }
-
-    private InboundChannelBuffer(Supplier pageSupplier) {
+    public InboundChannelBuffer(Supplier pageSupplier) {
         this.pageSupplier = pageSupplier;
         this.pages = new ArrayDeque<>();
         this.capacity = PAGE_SIZE * pages.size();
         ensureCapacity(PAGE_SIZE);
     }
 
+    @Override
+    public void close() {
+        if (isClosed.compareAndSet(false, true)) {
+            Page page;
+            List closingExceptions = new ArrayList<>();
+            while ((page = pages.pollFirst()) != null) {
+                try {
+                    page.close();
+                } catch (RuntimeException e) {
+                    closingExceptions.add(e);
+                }
+            }
+            ExceptionsHelper.rethrowAndSuppress(closingExceptions);
+        }
+    }
+
     public void ensureCapacity(long requiredCapacity) {
+        if (isClosed.get()) {
+            throw new IllegalStateException("Cannot allocate new pages if the buffer is closed.");
+        }
         if (capacity < requiredCapacity) {
             int numPages = numPages(requiredCapacity + offset);
             int pagesToAdd = numPages - pages.size();
             for (int i = 0; i < pagesToAdd; i++) {
-                pages.addLast(pageSupplier.get());
+                Page page = pageSupplier.get();
+                pages.addLast(page);
             }
             capacity += pagesToAdd * PAGE_SIZE;
         }
@@ -81,7 +105,7 @@ public final class InboundChannelBuffer {
 
         int pagesToRelease = pageIndex(offset + bytesToRelease);
         for (int i = 0; i < pagesToRelease; i++) {
-            pages.removeFirst();
+            pages.removeFirst().close();
         }
         capacity -= bytesToRelease;
         internalIndex = Math.max(internalIndex - bytesToRelease, 0);
@@ -112,12 +136,12 @@ public final class InboundChannelBuffer {
         }
 
         ByteBuffer[] buffers = new ByteBuffer[pageCount];
-        Iterator pageIterator = pages.iterator();
-        ByteBuffer firstBuffer = pageIterator.next().duplicate();
+        Iterator pageIterator = pages.iterator();
+        ByteBuffer firstBuffer = pageIterator.next().byteBuffer.duplicate();
         firstBuffer.position(firstBuffer.position() + offset);
         buffers[0] = firstBuffer;
         for (int i = 1; i < buffers.length; i++) {
-            buffers[i] = pageIterator.next().duplicate();
+            buffers[i] = pageIterator.next().byteBuffer.duplicate();
         }
         if (finalLimit != 0) {
             buffers[buffers.length - 1].limit(finalLimit);
@@ -148,11 +172,11 @@ public final class InboundChannelBuffer {
         int indexInPage = indexInPage(indexWithOffset);
 
         ByteBuffer[] buffers = new ByteBuffer[pages.size() - pageIndex];
-        Iterator pageIterator = pages.descendingIterator();
+        Iterator pageIterator = pages.descendingIterator();
         for (int i = buffers.length - 1; i > 0; --i) {
-            buffers[i] = pageIterator.next().duplicate();
+            buffers[i] = pageIterator.next().byteBuffer.duplicate();
         }
-        ByteBuffer firstPostIndexBuffer = pageIterator.next().duplicate();
+        ByteBuffer firstPostIndexBuffer = pageIterator.next().byteBuffer.duplicate();
         firstPostIndexBuffer.position(firstPostIndexBuffer.position() + indexInPage);
         buffers[0] = firstPostIndexBuffer;
 
@@ -201,4 +225,21 @@ public final class InboundChannelBuffer {
     private int indexInPage(long index) {
         return (int) (index & PAGE_MASK);
     }
+
+    public static class Page implements Releasable {
+
+        private final ByteBuffer byteBuffer;
+        private final Releasable releasable;
+
+        public Page(ByteBuffer byteBuffer, Releasable releasable) {
+            this.byteBuffer = byteBuffer;
+            this.releasable = releasable;
+        }
+
+        @Override
+        public void close() {
+            releasable.close();
+        }
+
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java
index d1ab10fb568..c39e4517b8b 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java
@@ -24,10 +24,12 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.recycler.Recycler;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -44,6 +46,7 @@ import org.elasticsearch.transport.nio.channel.TcpWriteContext;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ThreadFactory;
@@ -68,6 +71,7 @@ public class NioTransport extends TcpTransport {
         intSetting("transport.nio.acceptor_count", 1, 1, Setting.Property.NodeScope);
 
     private final OpenChannels openChannels = new OpenChannels(logger);
+    private final PageCacheRecycler pageCacheRecycler;
     private final ConcurrentMap profileToChannelFactory = newConcurrentMap();
     private final ArrayList acceptors = new ArrayList<>();
     private final ArrayList socketSelectors = new ArrayList<>();
@@ -76,8 +80,10 @@ public class NioTransport extends TcpTransport {
     private int acceptorNumber;
 
     public NioTransport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays,
-                        NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) {
+                        PageCacheRecycler pageCacheRecycler, NamedWriteableRegistry namedWriteableRegistry,
+                        CircuitBreakerService circuitBreakerService) {
         super("nio", settings, threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, networkService);
+        this.pageCacheRecycler = pageCacheRecycler;
     }
 
     @Override
@@ -184,8 +190,14 @@ public class NioTransport extends TcpTransport {
     }
 
     private Consumer getContextSetter(String profileName) {
-        return (c) -> c.setContexts(new TcpReadContext(c, new TcpReadHandler(profileName,this)), new TcpWriteContext(c),
-            this::exceptionCaught);
+        return (c) -> {
+            Supplier pageSupplier = () -> {
+                Recycler.V bytes = pageCacheRecycler.bytePage(false);
+                return new InboundChannelBuffer.Page(ByteBuffer.wrap(bytes.v()), bytes);
+            };
+            c.setContexts(new TcpReadContext(c, new TcpReadHandler(profileName, this), new InboundChannelBuffer(pageSupplier)),
+                new TcpWriteContext(c), this::exceptionCaught);
+        };
     }
 
     private void acceptChannel(NioSocketChannel channel) {
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransportPlugin.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransportPlugin.java
index 733351fe429..e158fe6fe97 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransportPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransportPlugin.java
@@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -38,6 +39,7 @@ public class NioTransportPlugin extends Plugin implements NetworkPlugin {
 
     @Override
     public Map> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                                                          PageCacheRecycler pageCacheRecycler,
                                                           CircuitBreakerService circuitBreakerService,
                                                           NamedWriteableRegistry namedWriteableRegistry,
                                                           NetworkService networkService) {
@@ -49,6 +51,7 @@ public class NioTransportPlugin extends Plugin implements NetworkPlugin {
             settings1 = settings;
         }
         return Collections.singletonMap(NIO_TRANSPORT_NAME,
-            () -> new NioTransport(settings1, threadPool, networkService, bigArrays, namedWriteableRegistry, circuitBreakerService));
+            () -> new NioTransport(settings1, threadPool, networkService, bigArrays, pageCacheRecycler, namedWriteableRegistry,
+                circuitBreakerService));
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/NioSocketChannel.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/NioSocketChannel.java
index 0f6c6715088..06f8aec6279 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/NioSocketChannel.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/NioSocketChannel.java
@@ -56,6 +56,7 @@ public class NioSocketChannel extends AbstractNioChannel {
         if (writeContext.hasQueuedWriteOps()) {
             writeContext.clearQueuedWriteOps(new ClosedChannelException());
         }
+        readContext.close();
 
         super.closeFromSelector();
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/ReadContext.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/ReadContext.java
index 9d2919b1928..243a6d8b239 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/ReadContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/ReadContext.java
@@ -21,8 +21,11 @@ package org.elasticsearch.transport.nio.channel;
 
 import java.io.IOException;
 
-public interface ReadContext {
+public interface ReadContext extends AutoCloseable {
 
     int read() throws IOException;
 
+    @Override
+    void close();
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpReadContext.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpReadContext.java
index ae9fe0fdc93..24a671412d2 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpReadContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/channel/TcpReadContext.java
@@ -32,17 +32,13 @@ public class TcpReadContext implements ReadContext {
 
     private final TcpReadHandler handler;
     private final TcpNioSocketChannel channel;
-    private final TcpFrameDecoder frameDecoder;
-    private final InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+    private final InboundChannelBuffer channelBuffer;
+    private final TcpFrameDecoder frameDecoder = new TcpFrameDecoder();
 
-    public TcpReadContext(NioSocketChannel channel, TcpReadHandler handler) {
-        this((TcpNioSocketChannel) channel, handler, new TcpFrameDecoder());
-    }
-
-    public TcpReadContext(TcpNioSocketChannel channel, TcpReadHandler handler, TcpFrameDecoder frameDecoder) {
+    public TcpReadContext(NioSocketChannel channel, TcpReadHandler handler, InboundChannelBuffer channelBuffer) {
         this.handler = handler;
-        this.channel = channel;
-        this.frameDecoder = frameDecoder;
+        this.channel = (TcpNioSocketChannel) channel;
+        this.channelBuffer = channelBuffer;
     }
 
     @Override
@@ -82,6 +78,11 @@ public class TcpReadContext implements ReadContext {
         return bytesRead;
     }
 
+    @Override
+    public void close() {
+        channelBuffer.close();
+    }
+
     private static BytesReference toBytesReference(InboundChannelBuffer channelBuffer) {
         ByteBuffer[] writtenToBuffers = channelBuffer.sliceBuffersTo(channelBuffer.getIndex());
         ByteBufferReference[] references = new ByteBufferReference[writtenToBuffers.length];
@@ -91,5 +92,4 @@ public class TcpReadContext implements ReadContext {
 
         return new CompositeBytesReference(references);
     }
-
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/InboundChannelBufferTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/InboundChannelBufferTests.java
index 7232a938710..9620f4c9c76 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/InboundChannelBufferTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/InboundChannelBufferTests.java
@@ -19,16 +19,22 @@
 
 package org.elasticsearch.transport.nio;
 
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.ByteBuffer;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
 
 public class InboundChannelBufferTests extends ESTestCase {
 
-    private static final int PAGE_SIZE = 1 << 14;
+    private static final int PAGE_SIZE = BigArrays.PAGE_SIZE_IN_BYTES;
+    private final Supplier defaultPageSupplier = () ->
+        new InboundChannelBuffer.Page(ByteBuffer.allocate(BigArrays.BYTE_PAGE_SIZE), () -> {});
 
     public void testNewBufferHasSinglePage() {
-        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer(defaultPageSupplier);
 
         assertEquals(PAGE_SIZE, channelBuffer.getCapacity());
         assertEquals(PAGE_SIZE, channelBuffer.getRemaining());
@@ -36,7 +42,7 @@ public class InboundChannelBufferTests extends ESTestCase {
     }
 
     public void testExpandCapacity() {
-        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer(defaultPageSupplier);
 
         assertEquals(PAGE_SIZE, channelBuffer.getCapacity());
         assertEquals(PAGE_SIZE, channelBuffer.getRemaining());
@@ -48,7 +54,7 @@ public class InboundChannelBufferTests extends ESTestCase {
     }
 
     public void testExpandCapacityMultiplePages() {
-        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer(defaultPageSupplier);
 
         assertEquals(PAGE_SIZE, channelBuffer.getCapacity());
 
@@ -60,7 +66,7 @@ public class InboundChannelBufferTests extends ESTestCase {
     }
 
     public void testExpandCapacityRespectsOffset() {
-        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer(defaultPageSupplier);
 
         assertEquals(PAGE_SIZE, channelBuffer.getCapacity());
         assertEquals(PAGE_SIZE, channelBuffer.getRemaining());
@@ -79,7 +85,7 @@ public class InboundChannelBufferTests extends ESTestCase {
     }
 
     public void testIncrementIndex() {
-        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer(defaultPageSupplier);
 
         assertEquals(0, channelBuffer.getIndex());
         assertEquals(PAGE_SIZE, channelBuffer.getRemaining());
@@ -91,7 +97,7 @@ public class InboundChannelBufferTests extends ESTestCase {
     }
 
     public void testIncrementIndexWithOffset() {
-        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer(defaultPageSupplier);
 
         assertEquals(0, channelBuffer.getIndex());
         assertEquals(PAGE_SIZE, channelBuffer.getRemaining());
@@ -109,8 +115,60 @@ public class InboundChannelBufferTests extends ESTestCase {
         assertEquals(PAGE_SIZE - 20, channelBuffer.getRemaining());
     }
 
+    public void testReleaseClosesPages() {
+        ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>();
+        Supplier supplier = () -> {
+            AtomicBoolean atomicBoolean = new AtomicBoolean();
+            queue.add(atomicBoolean);
+            return new InboundChannelBuffer.Page(ByteBuffer.allocate(PAGE_SIZE), () -> atomicBoolean.set(true));
+        };
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer(supplier);
+        channelBuffer.ensureCapacity(PAGE_SIZE * 4);
+
+        assertEquals(PAGE_SIZE * 4, channelBuffer.getCapacity());
+        assertEquals(4, queue.size());
+
+        for (AtomicBoolean closedRef : queue) {
+            assertFalse(closedRef.get());
+        }
+
+        channelBuffer.release(2 * PAGE_SIZE);
+
+        assertEquals(PAGE_SIZE * 2, channelBuffer.getCapacity());
+
+        assertTrue(queue.poll().get());
+        assertTrue(queue.poll().get());
+        assertFalse(queue.poll().get());
+        assertFalse(queue.poll().get());
+    }
+
+    public void testClose() {
+        ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>();
+        Supplier supplier = () -> {
+            AtomicBoolean atomicBoolean = new AtomicBoolean();
+            queue.add(atomicBoolean);
+            return new InboundChannelBuffer.Page(ByteBuffer.allocate(PAGE_SIZE), () -> atomicBoolean.set(true));
+        };
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer(supplier);
+        channelBuffer.ensureCapacity(PAGE_SIZE * 4);
+
+        assertEquals(4, queue.size());
+
+        for (AtomicBoolean closedRef : queue) {
+            assertFalse(closedRef.get());
+        }
+
+        channelBuffer.close();
+
+        for (AtomicBoolean closedRef : queue) {
+            assertTrue(closedRef.get());
+        }
+
+        expectThrows(IllegalStateException.class, () -> channelBuffer.ensureCapacity(1));
+    }
+
     public void testAccessByteBuffers() {
-        InboundChannelBuffer channelBuffer = new InboundChannelBuffer();
+        InboundChannelBuffer channelBuffer = new InboundChannelBuffer(defaultPageSupplier);
 
         int pages = randomInt(50) + 5;
         channelBuffer.ensureCapacity(pages * PAGE_SIZE);
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java
index 55bca45d1c8..1cff80dec79 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java
@@ -28,6 +28,8 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -57,8 +59,8 @@ public class SimpleNioTransportTests extends AbstractSimpleTransportTestCase {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         NetworkService networkService = new NetworkService(Collections.emptyList());
         Transport transport = new NioTransport(settings, threadPool,
-            networkService,
-            BigArrays.NON_RECYCLING_INSTANCE, namedWriteableRegistry, new NoneCircuitBreakerService()) {
+            networkService, BigArrays.NON_RECYCLING_INSTANCE, new MockPageCacheRecycler(settings), namedWriteableRegistry,
+            new NoneCircuitBreakerService()) {
 
             @Override
             protected Version executeHandshake(DiscoveryNode node, TcpChannel channel, TimeValue timeout) throws IOException,
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpReadContextTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpReadContextTests.java
index 73583353f73..f24c087e60a 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpReadContextTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/channel/TcpReadContextTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.transport.nio.channel;
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.CompositeBytesReference;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.nio.InboundChannelBuffer;
 import org.elasticsearch.transport.nio.TcpReadHandler;
@@ -30,6 +31,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
@@ -45,12 +47,14 @@ public class TcpReadContextTests extends ESTestCase {
     private TcpReadContext readContext;
 
     @Before
-    public void init() throws IOException {
+    public void init() {
         handler = mock(TcpReadHandler.class);
 
         messageLength = randomInt(96) + 4;
         channel = mock(TcpNioSocketChannel.class);
-        readContext = new TcpReadContext(channel, handler);
+        Supplier pageSupplier = () ->
+            new InboundChannelBuffer.Page(ByteBuffer.allocate(BigArrays.BYTE_PAGE_SIZE), () -> {});
+        readContext = new TcpReadContext(channel, handler, new InboundChannelBuffer(pageSupplier));
     }
 
     public void testSuccessfulRead() throws IOException {
@@ -122,6 +126,15 @@ public class TcpReadContextTests extends ESTestCase {
         }
     }
 
+    public void closeClosesChannelBuffer() {
+        InboundChannelBuffer buffer = mock(InboundChannelBuffer.class);
+        TcpReadContext readContext = new TcpReadContext(channel, handler, buffer);
+
+        readContext.close();
+
+        verify(buffer).close();
+    }
+
     private static byte[] combineMessageAndHeader(byte[] bytes) {
         return combineMessageAndHeader(bytes, bytes.length);
     }

From cbba37c17dd51a91b09715c00282546a58fb79b3 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Fri, 8 Dec 2017 13:51:38 -0500
Subject: [PATCH 226/297] Set ACK timeout on indices service test

Setting a timeout here speeds the test up significantly since we do not
need to wait up the default of 30 seconds for shards to start, we only
need an ACK that the index was opened.
---
 .../java/org/elasticsearch/indices/IndicesServiceTests.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index 01f1c3dab9c..7cef608850e 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -240,7 +240,7 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
             assertEquals(indicesService.numPendingDeletes(test.index()), 0);
             assertTrue(indicesService.hasUncompletedPendingDeletes()); // "bogus" index has not been removed
         }
-        assertAcked(client().admin().indices().prepareOpen("test"));
+        assertAcked(client().admin().indices().prepareOpen("test").setTimeout(TimeValue.timeValueSeconds(1)));
 
     }
 

From b66a0721da5632a7481db3dcf468fb7c82be07fc Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Fri, 8 Dec 2017 14:33:05 -0500
Subject: [PATCH 227/297] Do not open indices with broken settings

Today we are lenient and we open an index if it has broken
settings. This can happen if a user installs a plugin that registers an
index setting, creates an index with that setting, stop their node,
removes the plugin, and then restarts the node. In this case, the index
will have a setting that we do not recognize yet we open the index
anyway. This leniency is dangerous so this commit removes it. Note that
we still are lenient on upgrades and we should really reconsider this in
a follow-up.

Relates #26995
---
 .../metadata/MetaDataIndexUpgradeService.java |  1 -
 .../settings/AbstractScopedSettings.java      | 38 +++++++++++---
 .../common/settings/IndexScopedSettings.java  |  3 +-
 .../elasticsearch/indices/IndicesService.java | 10 ++--
 .../admin/indices/create/CreateIndexIT.java   | 37 ++++++++++++++
 .../index/IndexSettingsTests.java             | 50 +++++++++++++++++++
 6 files changed, 125 insertions(+), 14 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
index 2ff5fd5c2b2..d58ed04a930 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
@@ -82,7 +82,6 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
     public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {
         // Throws an exception if there are too-old segments:
         if (isUpgraded(indexMetaData)) {
-            assert indexMetaData == archiveBrokenIndexSettings(indexMetaData) : "all settings must have been upgraded before";
             return indexMetaData;
         }
         checkSupportedVersion(indexMetaData, minimumIndexCompatibilityVersion);
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index f952eb36a0d..e2f4d7697b6 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -264,17 +264,41 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
     }
 
     /**
-     * Validates that all given settings are registered and valid
-     * @param settings the settings to validate
-     * @param validateDependencies if true settings dependencies are validated as well.
+     * Validates that all settings are registered and valid.
+     *
+     * @param settings             the settings to validate
+     * @param validateDependencies true if dependent settings should be validated
      * @see Setting#getSettingsDependencies(String)
      */
-    public final void validate(Settings settings, boolean validateDependencies) {
-        List exceptions = new ArrayList<>();
-        for (String key : settings.keySet()) { // settings iterate in deterministic fashion
+    public final void validate(final Settings settings, final boolean validateDependencies) {
+        validate(settings, validateDependencies, false, false);
+    }
+
+    /**
+     * Validates that all settings are registered and valid.
+     *
+     * @param settings               the settings
+     * @param validateDependencies   true if dependent settings should be validated
+     * @param ignorePrivateSettings  true if private settings should be ignored during validation
+     * @param ignoreArchivedSettings true if archived settings should be ignored during validation
+     * @see Setting#getSettingsDependencies(String)
+     */
+    public final void validate(
+            final Settings settings,
+            final boolean validateDependencies,
+            final boolean ignorePrivateSettings,
+            final boolean ignoreArchivedSettings) {
+        final List exceptions = new ArrayList<>();
+        for (final String key : settings.keySet()) { // settings iterate in deterministic fashion
+            if (isPrivateSetting(key) && ignorePrivateSettings) {
+                continue;
+            }
+            if (key.startsWith(ARCHIVED_SETTINGS_PREFIX) && ignoreArchivedSettings) {
+                continue;
+            }
             try {
                 validate(key, settings, validateDependencies);
-            } catch (RuntimeException ex) {
+            } catch (final RuntimeException ex) {
                 exceptions.add(ex);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index ddbb8b83d13..83de9ae3779 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -169,8 +169,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
 
     )));
 
-    public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY,
-            BUILT_IN_INDEX_SETTINGS);
+    public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY, BUILT_IN_INDEX_SETTINGS);
 
     public IndexScopedSettings(Settings settings, Set> settingsSet) {
         super(settings, settingsSet, Property.IndexScope);
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 314b46b4d08..7e0bff53841 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -152,7 +152,7 @@ public class IndicesService extends AbstractLifecycleComponent
     private final TimeValue shardsClosedTimeout;
     private final AnalysisRegistry analysisRegistry;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
-    private final IndexScopedSettings indexScopeSetting;
+    private final IndexScopedSettings indexScopedSettings;
     private final IndicesFieldDataCache indicesFieldDataCache;
     private final CacheCleaner cacheCleaner;
     private final ThreadPool threadPool;
@@ -198,7 +198,7 @@ public class IndicesService extends AbstractLifecycleComponent
         indexingMemoryController = new IndexingMemoryController(settings, threadPool,
                                                                 // ensure we pull an iter with new shards - flatten makes a copy
                                                                 () -> Iterables.flatten(this).iterator());
-        this.indexScopeSetting = indexScopedSettings;
+        this.indexScopedSettings = indexScopedSettings;
         this.circuitBreakerService = circuitBreakerService;
         this.bigArrays = bigArrays;
         this.scriptService = scriptService;
@@ -432,7 +432,9 @@ public class IndicesService extends AbstractLifecycleComponent
                                                          IndicesFieldDataCache indicesFieldDataCache,
                                                          List builtInListeners,
                                                          IndexingOperationListener... indexingOperationListeners) throws IOException {
-        final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexScopeSetting);
+        final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexScopedSettings);
+        // we ignore private settings since they are not registered settings
+        indexScopedSettings.validate(indexMetaData.getSettings(), true, true, true);
         logger.debug("creating Index [{}], shards [{}]/[{}] - reason [{}]",
             indexMetaData.getIndex(),
             idxSettings.getNumberOfShards(),
@@ -470,7 +472,7 @@ public class IndicesService extends AbstractLifecycleComponent
      * Note: the returned {@link MapperService} should be closed when unneeded.
      */
     public synchronized MapperService createIndexMapperService(IndexMetaData indexMetaData) throws IOException {
-        final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexScopeSetting);
+        final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexScopedSettings);
         final IndexModule indexModule = new IndexModule(idxSettings, analysisRegistry);
         pluginsService.onIndexModule(indexModule);
         return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry, scriptService);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 5385902bc36..7477f11601f 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.create;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
@@ -32,6 +33,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -51,6 +53,8 @@ import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.core.IsNull.notNullValue;
 
@@ -344,4 +348,37 @@ public class CreateIndexIT extends ESIntegTestCase {
 
         assertEquals("Should have index name in response", "foo", response.index());
     }
+
+    public void testIndexWithUnknownSetting() throws Exception {
+        final int replicas = internalCluster().numDataNodes() - 1;
+        final Settings settings = Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", replicas).build();
+        client().admin().indices().prepareCreate("test").setSettings(settings).get();
+        ensureGreen("test");
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetaData metaData = state.getMetaData().index("test");
+        for (final NodeEnvironment services : internalCluster().getInstances(NodeEnvironment.class)) {
+            final IndexMetaData brokenMetaData =
+                    IndexMetaData
+                            .builder(metaData)
+                            .settings(Settings.builder().put(metaData.getSettings()).put("index.foo", "true"))
+                            .build();
+            // so evil
+            IndexMetaData.FORMAT.write(brokenMetaData, services.indexPaths(brokenMetaData.getIndex()));
+        }
+        internalCluster().fullRestart();
+        ensureGreen(metaData.getIndex().getName()); // we have to wait for the index to show up in the metadata or we will fail in a race
+        final ClusterState stateAfterRestart = client().admin().cluster().prepareState().get().getState();
+
+        // the index should not be open after we restart and recover the broken index metadata
+        assertThat(stateAfterRestart.getMetaData().index(metaData.getIndex()).getState(), equalTo(IndexMetaData.State.CLOSE));
+
+        // try to open the index
+        final ElasticsearchException e =
+                expectThrows(ElasticsearchException.class, () -> client().admin().indices().prepareOpen("test").get());
+        assertThat(e, hasToString(containsString("Failed to verify index " + metaData.getIndex())));
+        assertNotNull(e.getCause());
+        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
+        assertThat(e, hasToString(containsString("unknown setting [index.foo]")));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
index 79c306f43f1..1149111f613 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.AbstractScopedSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -444,6 +445,55 @@ public class IndexSettingsTests extends ESTestCase {
         assertEquals(actual, settings.getGenerationThresholdSize());
     }
 
+    public void testPrivateSettingsValidation() {
+        final Settings settings = Settings.builder().put(IndexMetaData.SETTING_CREATION_DATE, System.currentTimeMillis()).build();
+        final IndexScopedSettings indexScopedSettings = new IndexScopedSettings(settings, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
+
+        {
+            // validation should fail since we are not ignoring private settings
+            final IllegalArgumentException e = expectThrows(
+                    IllegalArgumentException.class,
+                    () -> indexScopedSettings.validate(settings, randomBoolean()));
+            assertThat(e, hasToString(containsString("unknown setting [index.creation_date]")));
+        }
+
+        {
+            // validation should fail since we are not ignoring private settings
+            final IllegalArgumentException e = expectThrows(
+                    IllegalArgumentException.class,
+                    () -> indexScopedSettings.validate(settings, randomBoolean(), false, randomBoolean()));
+            assertThat(e, hasToString(containsString("unknown setting [index.creation_date]")));
+        }
+
+        // nothing should happen since we are ignoring private settings
+        indexScopedSettings.validate(settings, randomBoolean(), true, randomBoolean());
+    }
+
+    public void testArchivedSettingsValidation() {
+        final Settings settings =
+                Settings.builder().put(AbstractScopedSettings.ARCHIVED_SETTINGS_PREFIX + "foo", System.currentTimeMillis()).build();
+        final IndexScopedSettings indexScopedSettings = new IndexScopedSettings(settings, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
+
+        {
+            // validation should fail since we are not ignoring archived settings
+            final IllegalArgumentException e = expectThrows(
+                    IllegalArgumentException.class,
+                    () -> indexScopedSettings.validate(settings, randomBoolean()));
+            assertThat(e, hasToString(containsString("unknown setting [archived.foo]")));
+        }
+
+        {
+            // validation should fail since we are not ignoring archived settings
+            final IllegalArgumentException e = expectThrows(
+                    IllegalArgumentException.class,
+                    () -> indexScopedSettings.validate(settings, randomBoolean(), randomBoolean(), false));
+            assertThat(e, hasToString(containsString("unknown setting [archived.foo]")));
+        }
+
+        // nothing should happen since we are ignoring archived settings
+        indexScopedSettings.validate(settings, randomBoolean(), randomBoolean(), true);
+    }
+
     public void testArchiveBrokenIndexSettings() {
         Settings settings =
             IndexScopedSettings.DEFAULT_SCOPED_SETTINGS.archiveUnknownOrInvalidSettings(

From f50f99ef111b6253bbdb9fac6d88143c236f4a0c Mon Sep 17 00:00:00 2001
From: olcbean <26058559+olcbean@users.noreply.github.com>
Date: Fri, 8 Dec 2017 21:46:56 +0100
Subject: [PATCH 228/297] Improve error msg when a field name contains only
 white spaces (#27709)

* Explicitly check if a field name contains only
white spaces

* "white spaces" changed to "whitespace"
---
 .../index/mapper/DocumentParser.java            |  5 +++++
 .../index/mapper/DocumentParserTests.java       | 17 +++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index dd3aa69315d..596581a15a2 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -180,6 +180,11 @@ final class DocumentParser {
             String[] parts = fullFieldPath.split("\\.");
             for (String part : parts) {
                 if (Strings.hasText(part) == false) {
+                    // check if the field name contains only whitespace
+                    if (Strings.isEmpty(part) == false) {
+                        throw new IllegalArgumentException(
+                                "object field cannot contain only whitespace: ['" + fullFieldPath + "']");
+                    }
                     throw new IllegalArgumentException(
                             "object field starting or ending with a [.] makes object resolution ambiguous: [" + fullFieldPath + "]");
                 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index cbf890ef476..b3b33cf0dd6 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -1377,6 +1377,23 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
         }
     }
 
+    public void testDynamicFieldsEmptyName() throws Exception {
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().startArray("top.")
+                .startObject()
+                .startObject("aoeu")
+                .field("a", 1).field(" ", 2)
+                .endObject()
+                .endObject().endArray()
+                .endObject().bytes();
+
+        IllegalArgumentException emptyFieldNameException = expectThrows(IllegalArgumentException.class,
+                () -> client().prepareIndex("idx", "type").setSource(bytes, XContentType.JSON).get());
+
+        assertThat(emptyFieldNameException.getMessage(), containsString(
+                "object field cannot contain only whitespace: ['top.aoeu. ']"));
+    }
+
     public void testBlankFieldNames() throws Exception {
         final BytesReference bytes = XContentFactory.jsonBuilder()
                 .startObject()

From d1acb7697b962429252213ddb146f10d3da86d65 Mon Sep 17 00:00:00 2001
From: Tim Brooks 
Date: Fri, 8 Dec 2017 16:56:53 -0700
Subject: [PATCH 229/297] Remove internal channel tracking in transports
 (#27711)

This commit attempts to continue unifying the logic between different
transport implementations. As transports call a `TcpTransport` callback
when a new channel is accepted, there is no need to internally track
channels accepted. Instead there is a set of accepted channels in
`TcpTransport`. This set is used for metrics and shutting down channels.
---
 .../elasticsearch/transport/TcpTransport.java |  28 ++---
 .../transport/TcpTransportTests.java          |   5 -
 .../transport/netty4/Netty4Transport.java     |  13 +-
 .../transport/netty4/NettyTcpChannel.java     |  10 +-
 .../AbstractSimpleTransportTestCase.java      |  22 ++++
 .../transport/MockTcpTransport.java           |   5 -
 .../transport/nio/NioShutdown.java            |   7 +-
 .../transport/nio/NioTransport.java           |  18 +--
 .../transport/nio/OpenChannels.java           | 116 ------------------
 .../transport/MockTcpTransportTests.java      |   5 +
 10 files changed, 52 insertions(+), 177 deletions(-)
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/transport/nio/OpenChannels.java

diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
index 6b532a600a1..5fbc45a804c 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -195,22 +195,22 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
     protected final NetworkService networkService;
     protected final Set profileSettings;
 
-    protected volatile TransportService transportService;
-    // node id to actual channel
-    protected final ConcurrentMap connectedNodes = newConcurrentMap();
+    private volatile TransportService transportService;
 
-    protected final ConcurrentMap profileBoundAddresses = newConcurrentMap();
+    private final ConcurrentMap profileBoundAddresses = newConcurrentMap();
+    // node id to actual channel
+    private final ConcurrentMap connectedNodes = newConcurrentMap();
     private final Map> serverChannels = newConcurrentMap();
     private final Set acceptedChannels = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
-    protected final KeyedLock connectionLock = new KeyedLock<>();
+    private final KeyedLock connectionLock = new KeyedLock<>();
     private final NamedWriteableRegistry namedWriteableRegistry;
 
     // this lock is here to make sure we close this transport and disconnect all the client nodes
     // connections while no connect operations is going on... (this might help with 100% CPU when stopping the transport?)
-    protected final ReadWriteLock closeLock = new ReentrantReadWriteLock();
+    private final ReadWriteLock closeLock = new ReentrantReadWriteLock();
     protected final boolean compress;
-    protected volatile BoundTransportAddress boundAddress;
+    private volatile BoundTransportAddress boundAddress;
     private final String transportName;
     protected final ConnectionProfile defaultConnectionProfile;
 
@@ -438,7 +438,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
         }
 
         @Override
-        public void close() throws IOException {
+        public void close() {
             if (closed.compareAndSet(false, true)) {
                 try {
                     if (lifecycle.stopped()) {
@@ -582,7 +582,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
     }
 
     @Override
-    public final NodeChannels openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) throws IOException {
+    public final NodeChannels openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) {
         if (node == null) {
             throw new ConnectTransportException(null, "can't open connection to a null node");
         }
@@ -602,6 +602,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
                         PlainActionFuture connectFuture = PlainActionFuture.newFuture();
                         connectionFutures.add(connectFuture);
                         TcpChannel channel = initiateChannel(node, connectionProfile.getConnectTimeout(), connectFuture);
+                        logger.trace(() -> new ParameterizedMessage("Tcp transport client channel opened: {}", channel));
                         channels.add(channel);
                     } catch (Exception e) {
                         // If there was an exception when attempting to instantiate the raw channels, we close all of the channels
@@ -1041,6 +1042,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
         boolean addedOnThisCall = acceptedChannels.add(channel);
         assert addedOnThisCall : "Channel should only be added to accept channel set once";
         channel.addCloseListener(ActionListener.wrap(() -> acceptedChannels.remove(channel)));
+        logger.trace(() -> new ParameterizedMessage("Tcp transport channel accepted: {}", channel));
     }
 
     /**
@@ -1738,15 +1740,9 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
         }
     }
 
-    /**
-     * Returns count of currently open connections
-     */
-    protected abstract long getNumOpenServerConnections();
-
     @Override
     public final TransportStats getStats() {
-        return new TransportStats(
-            getNumOpenServerConnections(), readBytesMetric.count(), readBytesMetric.sum(), transmittedBytesMetric.count(),
+        return new TransportStats(acceptedChannels.size(), readBytesMetric.count(), readBytesMetric.sum(), transmittedBytesMetric.count(),
             transmittedBytesMetric.sum());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/transport/TcpTransportTests.java b/core/src/test/java/org/elasticsearch/transport/TcpTransportTests.java
index 56fe7a5ebec..059f82e3603 100644
--- a/core/src/test/java/org/elasticsearch/transport/TcpTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/TcpTransportTests.java
@@ -191,11 +191,6 @@ public class TcpTransportTests extends ESTestCase {
                     return new FakeChannel(messageCaptor);
                 }
 
-                @Override
-                public long getNumOpenServerConnections() {
-                    return 0;
-                }
-
                 @Override
                 public NodeChannels getConnection(DiscoveryNode node) {
                     int numConnections = MockTcpTransport.LIGHT_PROFILE.getNumConnections();
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
index 6343a241843..87a09ad8ee3 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
@@ -104,8 +104,6 @@ public class Netty4Transport extends TcpTransport {
     protected final int workerCount;
     protected final ByteSizeValue receivePredictorMin;
     protected final ByteSizeValue receivePredictorMax;
-    // package private for testing
-    volatile Netty4OpenChannelsHandler serverOpenChannels;
     protected volatile Bootstrap bootstrap;
     protected final Map serverBootstraps = newConcurrentMap();
 
@@ -132,8 +130,6 @@ public class Netty4Transport extends TcpTransport {
         try {
             bootstrap = createBootstrap();
             if (NetworkService.NETWORK_SERVER.get(settings)) {
-                final Netty4OpenChannelsHandler openChannels = new Netty4OpenChannelsHandler(logger);
-                this.serverOpenChannels = openChannels;
                 for (ProfileSettings profileSettings : profileSettings) {
                     createServerBootstrap(profileSettings);
                     bindServer(profileSettings);
@@ -242,12 +238,6 @@ public class Netty4Transport extends TcpTransport {
         onException(channel.attr(CHANNEL_KEY).get(), t instanceof Exception ? (Exception) t : new ElasticsearchException(t));
     }
 
-    @Override
-    public long getNumOpenServerConnections() {
-        Netty4OpenChannelsHandler channels = serverOpenChannels;
-        return channels == null ? 0 : channels.numberOfOpenChannels();
-    }
-
     @Override
     protected NettyTcpChannel initiateChannel(DiscoveryNode node, TimeValue connectTimeout, ActionListener listener)
         throws IOException {
@@ -294,7 +284,7 @@ public class Netty4Transport extends TcpTransport {
     @Override
     @SuppressForbidden(reason = "debug")
     protected void stopInternal() {
-        Releasables.close(serverOpenChannels, () -> {
+        Releasables.close(() -> {
             final List>> serverBootstrapCloseFutures = new ArrayList<>(serverBootstraps.size());
             for (final Map.Entry entry : serverBootstraps.entrySet()) {
                 serverBootstrapCloseFutures.add(
@@ -349,7 +339,6 @@ public class Netty4Transport extends TcpTransport {
             ch.attr(CHANNEL_KEY).set(nettyTcpChannel);
             serverAcceptedChannel(nettyTcpChannel);
             ch.pipeline().addLast("logging", new ESLoggingHandler());
-            ch.pipeline().addLast("open_channels", Netty4Transport.this.serverOpenChannels);
             ch.pipeline().addLast("size", new Netty4SizeHeaderFrameDecoder());
             ch.pipeline().addLast("dispatcher", new Netty4MessageChannelHandler(Netty4Transport.this, name));
         }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyTcpChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyTcpChannel.java
index 3d71735a2a8..17c18f15ae1 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyTcpChannel.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyTcpChannel.java
@@ -96,7 +96,7 @@ public class NettyTcpChannel implements TcpChannel {
             }
         });
         channel.writeAndFlush(Netty4Utils.toByteBuf(reference), writePromise);
-        
+
         if (channel.eventLoop().isShutdown()) {
             listener.onFailure(new TransportException("Cannot send message, event loop is shutting down."));
         }
@@ -105,4 +105,12 @@ public class NettyTcpChannel implements TcpChannel {
     public Channel getLowLevelChannel() {
         return channel;
     }
+
+    @Override
+    public String toString() {
+        return "NettyTcpChannel{" +
+            "localAddress=" + getLocalAddress() +
+            ", remoteAddress=" + channel.remoteAddress() +
+            '}';
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index a45411324b0..17423207d56 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -109,6 +109,10 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
     protected abstract MockTransportService build(Settings settings, Version version, ClusterSettings clusterSettings, boolean doHandshake);
 
+    protected int channelsPerNodeConnection() {
+        return 13;
+    }
+
     @Override
     @Before
     public void setUp() throws Exception {
@@ -2345,6 +2349,24 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         }
     }
 
+    public void testAcceptedChannelCount() throws Exception {
+        assertBusy(() -> {
+            TransportStats transportStats = serviceA.transport.getStats();
+            assertEquals(channelsPerNodeConnection(), transportStats.getServerOpen());
+        });
+        assertBusy(() -> {
+            TransportStats transportStats = serviceB.transport.getStats();
+            assertEquals(channelsPerNodeConnection(), transportStats.getServerOpen());
+        });
+
+        serviceA.close();
+
+        assertBusy(() -> {
+            TransportStats transportStats = serviceB.transport.getStats();
+            assertEquals(0, transportStats.getServerOpen());
+        });
+    }
+
     public void testTransportStatsWithException() throws Exception {
         MockTransportService serviceC = build(Settings.builder().put("name", "TS_TEST").build(), version0, null, true);
         CountDownLatch receivedLatch = new CountDownLatch(1);
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java
index f6ec96d13df..68f79b1cef7 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java
@@ -217,11 +217,6 @@ public class MockTcpTransport extends TcpTransport {
         socket.setReuseAddress(TCP_REUSE_ADDRESS.get(settings));
     }
 
-    @Override
-    public long getNumOpenServerConnections() {
-        return 1;
-    }
-
     public final class MockChannel implements Closeable, TcpChannel {
         private final AtomicBoolean isOpen = new AtomicBoolean(true);
         private final InetSocketAddress localAddress;
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioShutdown.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioShutdown.java
index 320da6a88d1..3970e69b2c1 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioShutdown.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioShutdown.java
@@ -34,17 +34,12 @@ public class NioShutdown {
         this.logger = logger;
     }
 
-    void orderlyShutdown(OpenChannels openChannels, ArrayList acceptors, ArrayList socketSelectors) {
-
-        // Start by closing the server channels. Once these are closed, we are guaranteed to no accept new connections
-        openChannels.closeServerChannels();
+    void orderlyShutdown(ArrayList acceptors, ArrayList socketSelectors) {
 
         for (AcceptingSelector acceptor : acceptors) {
             shutdownSelector(acceptor);
         }
 
-        openChannels.close();
-
         for (SocketSelector selector : socketSelectors) {
             shutdownSelector(selector);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java
index c39e4517b8b..775030bc6db 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java
@@ -35,7 +35,6 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transports;
-import org.elasticsearch.transport.nio.channel.NioChannel;
 import org.elasticsearch.transport.nio.channel.NioServerSocketChannel;
 import org.elasticsearch.transport.nio.channel.NioSocketChannel;
 import org.elasticsearch.transport.nio.channel.TcpChannelFactory;
@@ -70,7 +69,6 @@ public class NioTransport extends TcpTransport {
     public static final Setting NIO_ACCEPTOR_COUNT =
         intSetting("transport.nio.acceptor_count", 1, 1, Setting.Property.NodeScope);
 
-    private final OpenChannels openChannels = new OpenChannels(logger);
     private final PageCacheRecycler pageCacheRecycler;
     private final ConcurrentMap profileToChannelFactory = newConcurrentMap();
     private final ArrayList acceptors = new ArrayList<>();
@@ -86,27 +84,17 @@ public class NioTransport extends TcpTransport {
         this.pageCacheRecycler = pageCacheRecycler;
     }
 
-    @Override
-    public long getNumOpenServerConnections() {
-        return openChannels.serverChannelsCount();
-    }
-
     @Override
     protected TcpNioServerSocketChannel bind(String name, InetSocketAddress address) throws IOException {
         TcpChannelFactory channelFactory = this.profileToChannelFactory.get(name);
         AcceptingSelector selector = acceptors.get(++acceptorNumber % NioTransport.NIO_ACCEPTOR_COUNT.get(settings));
-        TcpNioServerSocketChannel serverChannel = channelFactory.openNioServerSocketChannel(address, selector);
-        openChannels.serverChannelOpened(serverChannel);
-        serverChannel.addCloseListener(ActionListener.wrap(() -> openChannels.channelClosed(serverChannel)));
-        return serverChannel;
+        return channelFactory.openNioServerSocketChannel(address, selector);
     }
 
     @Override
     protected TcpNioSocketChannel initiateChannel(DiscoveryNode node, TimeValue connectTimeout, ActionListener connectListener)
         throws IOException {
         TcpNioSocketChannel channel = clientChannelFactory.openNioChannel(node.getAddress().address(), clientSelectorSupplier.get());
-        openChannels.clientChannelOpened(channel);
-        channel.addCloseListener(ActionListener.wrap(() -> openChannels.channelClosed(channel)));
         channel.addConnectListener(connectListener);
         return channel;
     }
@@ -175,7 +163,7 @@ public class NioTransport extends TcpTransport {
     @Override
     protected void stopInternal() {
         NioShutdown nioShutdown = new NioShutdown(logger);
-        nioShutdown.orderlyShutdown(openChannels, acceptors, socketSelectors);
+        nioShutdown.orderlyShutdown(acceptors, socketSelectors);
 
         profileToChannelFactory.clear();
         socketSelectors.clear();
@@ -202,8 +190,6 @@ public class NioTransport extends TcpTransport {
 
     private void acceptChannel(NioSocketChannel channel) {
         TcpNioSocketChannel tcpChannel = (TcpNioSocketChannel) channel;
-        openChannels.acceptedChannelOpened(tcpChannel);
-        tcpChannel.addCloseListener(ActionListener.wrap(() -> openChannels.channelClosed(channel)));
         serverAcceptedChannel(tcpChannel);
 
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/OpenChannels.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/OpenChannels.java
deleted file mode 100644
index 12c12aaa48e..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/OpenChannels.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.nio;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.lease.Releasable;
-import org.elasticsearch.transport.TcpChannel;
-import org.elasticsearch.transport.nio.channel.NioChannel;
-import org.elasticsearch.transport.nio.channel.NioServerSocketChannel;
-import org.elasticsearch.transport.nio.channel.NioSocketChannel;
-import org.elasticsearch.transport.nio.channel.TcpNioServerSocketChannel;
-import org.elasticsearch.transport.nio.channel.TcpNioSocketChannel;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.concurrent.ConcurrentMap;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
-
-public class OpenChannels implements Releasable {
-
-    // TODO: Maybe set concurrency levels?
-    private final ConcurrentMap openClientChannels = newConcurrentMap();
-    private final ConcurrentMap openAcceptedChannels = newConcurrentMap();
-    private final ConcurrentMap openServerChannels = newConcurrentMap();
-
-    private final Logger logger;
-
-    public OpenChannels(Logger logger) {
-        this.logger = logger;
-    }
-
-    public void serverChannelOpened(TcpNioServerSocketChannel channel) {
-        boolean added = openServerChannels.putIfAbsent(channel, System.nanoTime()) == null;
-        if (added && logger.isTraceEnabled()) {
-            logger.trace("server channel opened: {}", channel);
-        }
-    }
-
-    public long serverChannelsCount() {
-        return openServerChannels.size();
-    }
-
-    public void acceptedChannelOpened(TcpNioSocketChannel channel) {
-        boolean added = openAcceptedChannels.putIfAbsent(channel, System.nanoTime()) == null;
-        if (added && logger.isTraceEnabled()) {
-            logger.trace("accepted channel opened: {}", channel);
-        }
-    }
-
-    public HashSet getAcceptedChannels() {
-        return new HashSet<>(openAcceptedChannels.keySet());
-    }
-
-    public void clientChannelOpened(TcpNioSocketChannel channel) {
-        boolean added = openClientChannels.putIfAbsent(channel, System.nanoTime()) == null;
-        if (added && logger.isTraceEnabled()) {
-            logger.trace("client channel opened: {}", channel);
-        }
-    }
-
-    public Map getClientChannels() {
-        return openClientChannels;
-    }
-
-    public void channelClosed(NioChannel channel) {
-        boolean removed;
-        if (channel instanceof NioServerSocketChannel) {
-            removed = openServerChannels.remove(channel) != null;
-        } else {
-            NioSocketChannel socketChannel = (NioSocketChannel) channel;
-            removed = openClientChannels.remove(socketChannel) != null;
-            if (removed == false) {
-                removed = openAcceptedChannels.remove(socketChannel) != null;
-            }
-        }
-        if (removed && logger.isTraceEnabled()) {
-            logger.trace("channel closed: {}", channel);
-        }
-    }
-
-    public void closeServerChannels() {
-        TcpChannel.closeChannels(new ArrayList<>(openServerChannels.keySet()), true);
-
-        openServerChannels.clear();
-    }
-
-    @Override
-    public void close() {
-        Stream channels = Stream.concat(openClientChannels.keySet().stream(), openAcceptedChannels.keySet().stream());
-        TcpChannel.closeChannels(channels.collect(Collectors.toList()), true);
-
-        openClientChannels.clear();
-        openAcceptedChannels.clear();
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java
index 6844b55cadc..916e97ffd12 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/MockTcpTransportTests.java
@@ -55,6 +55,11 @@ public class MockTcpTransportTests extends AbstractSimpleTransportTestCase {
         return mockTransportService;
     }
 
+    @Override
+    public int channelsPerNodeConnection() {
+        return 1;
+    }
+
     @Override
     protected void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException {
         final MockTcpTransport t = (MockTcpTransport) transport;

From 8c8b1dc2cf5a480508f90e6e06fe1418a455e81c Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Sat, 9 Dec 2017 09:12:40 -0500
Subject: [PATCH 230/297] Fix index with unknown setting test

This commit fixes the test of an index with an unknown setting. The
problem here is that we were manipulating the index state on disk, but a
cluster state update could arrive between us manipulating the index
state on disk and us restarting the node, leading to the index state
that we just intentionally broke being fixed. As such, after restart,
the index state would not be in the state that we expected it to be in
and the test would fail. To address this, we hook into the restart and
break the index state immediately before the node is started again.

Relates #26995
---
 .../admin/indices/create/CreateIndexIT.java   | 37 ++++++++++++++-----
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 7477f11601f..3f99e437393 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.create;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.UnavailableShardsException;
@@ -30,6 +31,7 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -39,8 +41,11 @@ import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.test.InternalTestCluster;
 
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiFunction;
@@ -355,17 +360,29 @@ public class CreateIndexIT extends ESIntegTestCase {
         client().admin().indices().prepareCreate("test").setSettings(settings).get();
         ensureGreen("test");
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
-        final IndexMetaData metaData = state.getMetaData().index("test");
-        for (final NodeEnvironment services : internalCluster().getInstances(NodeEnvironment.class)) {
-            final IndexMetaData brokenMetaData =
-                    IndexMetaData
-                            .builder(metaData)
-                            .settings(Settings.builder().put(metaData.getSettings()).put("index.foo", "true"))
-                            .build();
-            // so evil
-            IndexMetaData.FORMAT.write(brokenMetaData, services.indexPaths(brokenMetaData.getIndex()));
+
+        final Set dataOrMasterNodeNames = new HashSet<>();
+        for (final ObjectCursor node : state.nodes().getMasterAndDataNodes().values()) {
+            assertTrue(dataOrMasterNodeNames.add(node.value.getName()));
         }
-        internalCluster().fullRestart();
+
+        final IndexMetaData metaData = state.getMetaData().index("test");
+        internalCluster().fullRestart(new InternalTestCluster.RestartCallback() {
+            @Override
+            public Settings onNodeStopped(String nodeName) throws Exception {
+                if (dataOrMasterNodeNames.contains(nodeName)) {
+                    final NodeEnvironment nodeEnvironment = internalCluster().getInstance(NodeEnvironment.class, nodeName);
+                    final IndexMetaData brokenMetaData =
+                            IndexMetaData
+                                    .builder(metaData)
+                                    .settings(Settings.builder().put(metaData.getSettings()).put("index.foo", true))
+                                    .build();
+                    // so evil
+                    IndexMetaData.FORMAT.write(brokenMetaData, nodeEnvironment.indexPaths(brokenMetaData.getIndex()));
+                }
+                return Settings.EMPTY;
+            }
+        });
         ensureGreen(metaData.getIndex().getName()); // we have to wait for the index to show up in the metadata or we will fail in a race
         final ClusterState stateAfterRestart = client().admin().cluster().prepareState().get().getState();
 

From 87f7b9c0f967d2514d26a0f11dbe601eab284327 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Sun, 10 Dec 2017 13:04:22 -0500
Subject: [PATCH 231/297] Speed up rejected execution contains node name test

This commit addresses slowness in the test that a rejected execution
contains the node name. The slowness came from setting the count on a
countdown latch too high (two in the case of the search thread pool)
where there would never be a second countdown on the latch. This means
that when then test node is shutting down, closing the node would have
to wait a full ten seconds before forcefully terminating the thread
pool. This commit fixes the issue so that the node can close
immediately, shaving ten seconds off the run time of the test.

Relates #27663
---
 .../common/util/concurrent/EsThreadPoolExecutorTests.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutorTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutorTests.java
index 9b9aa50bd16..b6110a85ece 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutorTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/EsThreadPoolExecutorTests.java
@@ -50,7 +50,7 @@ public class EsThreadPoolExecutorTests extends ESSingleNodeTestCase {
     }
 
     private void runThreadPoolExecutorTest(final int fill, final String executor) {
-        final CountDownLatch latch = new CountDownLatch(fill);
+        final CountDownLatch latch = new CountDownLatch(1);
         for (int i = 0; i < fill; i++) {
             node().injector().getInstance(ThreadPool.class).executor(executor).execute(() -> {
                 try {
@@ -64,12 +64,12 @@ public class EsThreadPoolExecutorTests extends ESSingleNodeTestCase {
         final AtomicBoolean rejected = new AtomicBoolean();
         node().injector().getInstance(ThreadPool.class).executor(executor).execute(new AbstractRunnable() {
             @Override
-            public void onFailure(Exception e) {
+            public void onFailure(final Exception e) {
 
             }
 
             @Override
-            public void onRejection(Exception e) {
+            public void onRejection(final Exception e) {
                 rejected.set(true);
                 assertThat(e, hasToString(containsString("name = es-thread-pool-executor-tests/" + executor + ", ")));
             }

From 87313e12ba76e9bd8b9be67d33deebbc8eaa6792 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?=
 <10398885+cbuescher@users.noreply.github.com>
Date: Mon, 11 Dec 2017 11:03:13 +0100
Subject: [PATCH 232/297] Use typeName() to check field type in
 GeoShapeQueryBuilder (#27730)

The current code contains an instanceOf check and a comment that this should
eventually be changed to something else. The typeName() should return a unique
name for the field type in question (geo_shape) so it can be used instead.
---
 .../elasticsearch/index/query/GeoShapeQueryBuilder.java  | 8 +++-----
 .../index/query/GeoShapeQueryBuilderTests.java           | 9 +++++++++
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index 0424cf6f14b..0f9006ed7da 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -341,11 +341,9 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder queryBuilder.toQuery(createShardContext()));
+        assertThat(e.getMessage(), containsString("Field [mapped_string] is not of type [geo_shape] but of type [text]"));
+    }
+
     public void testSerializationFailsUnlessFetched() throws IOException {
         QueryBuilder builder = doCreateTestQueryBuilder(true);
         QueryBuilder queryBuilder = Rewriteable.rewrite(builder, createShardContext());

From 85dd1880fc5e73c6cba4deb8b14e4cf0fafb9471 Mon Sep 17 00:00:00 2001
From: Robin Neatherway 
Date: Mon, 11 Dec 2017 10:28:03 +0000
Subject: [PATCH 233/297] Fix some type checks that were always false (#27706)

* CustomFieldQuery: removed a redundant type check that was
already done higher up in the same if/else chain.
* PrioritizedEsThreadPoolExecutor: removed a check that was
simply a duplicate of one earlier one and would never have been true.
---
 .../lucene/search/vectorhighlight/CustomFieldQuery.java      | 3 ---
 .../util/concurrent/PrioritizedEsThreadPoolExecutor.java     | 5 +----
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
index 5e877fbce40..1ace6fc34ba 100644
--- a/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
+++ b/core/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
@@ -75,9 +75,6 @@ public class CustomFieldQuery extends FieldQuery {
         } else if (sourceQuery instanceof BlendedTermQuery) {
             final BlendedTermQuery blendedTermQuery = (BlendedTermQuery) sourceQuery;
             flatten(blendedTermQuery.rewrite(reader), reader, flatQueries, boost);
-        } else if (sourceQuery instanceof ESToParentBlockJoinQuery) {
-            ESToParentBlockJoinQuery blockJoinQuery = (ESToParentBlockJoinQuery) sourceQuery;
-            flatten(blockJoinQuery.getChildQuery(), reader, flatQueries, boost);
         } else if (sourceQuery instanceof BoostingQuery) {
             BoostingQuery boostingQuery = (BoostingQuery) sourceQuery;
             //flatten positive query with query boost
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java
index ee38637b04c..8f9245ad583 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java
@@ -137,7 +137,7 @@ public class PrioritizedEsThreadPoolExecutor extends EsThreadPoolExecutor {
     @Override
     protected Runnable wrapRunnable(Runnable command) {
         if (command instanceof PrioritizedRunnable) {
-            if ((command instanceof TieBreakingPrioritizedRunnable)) {
+            if (command instanceof TieBreakingPrioritizedRunnable) {
                 return command;
             }
             Priority priority = ((PrioritizedRunnable) command).priority();
@@ -145,9 +145,6 @@ public class PrioritizedEsThreadPoolExecutor extends EsThreadPoolExecutor {
         } else if (command instanceof PrioritizedFutureTask) {
             return command;
         } else { // it might be a callable wrapper...
-            if (command instanceof TieBreakingPrioritizedRunnable) {
-                return command;
-            }
             return new TieBreakingPrioritizedRunnable(super.wrapRunnable(command), Priority.NORMAL, insertionOrder.incrementAndGet());
         }
     }

From 25c606cf09a3e5b03cce0340ee19553c5f559fdc Mon Sep 17 00:00:00 2001
From: olcbean <26058559+olcbean@users.noreply.github.com>
Date: Mon, 11 Dec 2017 13:16:04 +0100
Subject: [PATCH 234/297] Remove deprecated names for string distance
 algorithms (#27640)

#27409 deprecated the incorrectly-spelled `levenstein` in favour of `levenshtein`.
#27526 deprecated the inconsistent `jarowinkler` in favour of `jaro_winkler`.

These changes were merged into 6.2, and this change removes them entirely in 7.0.
---
 .../phrase/DirectCandidateGeneratorBuilder.java     | 10 ++--------
 .../search/suggest/term/TermSuggestionBuilder.java  |  9 +--------
 .../phrase/DirectCandidateGeneratorTests.java       | 10 ----------
 .../suggest/term/StringDistanceImplTests.java       | 11 -----------
 .../reference/migration/migrate_7_0/search.asciidoc | 13 +++++++++++--
 5 files changed, 14 insertions(+), 39 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
index 9631b37e2cd..5e0ed7749f7 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
@@ -464,19 +464,13 @@ public final class DirectCandidateGeneratorBuilder implements CandidateGenerator
     }
 
     static StringDistance resolveDistance(String distanceVal) {
-        distanceVal = distanceVal.toLowerCase(Locale.US);
+        distanceVal = distanceVal.toLowerCase(Locale.ROOT);
         if ("internal".equals(distanceVal)) {
             return DirectSpellChecker.INTERNAL_LEVENSHTEIN;
-        } else if ("damerau_levenshtein".equals(distanceVal) || "damerauLevenshtein".equals(distanceVal)) {
+        } else if ("damerau_levenshtein".equals(distanceVal)) {
             return new LuceneLevenshteinDistance();
-        } else if ("levenstein".equals(distanceVal)) {
-            DEPRECATION_LOGGER.deprecated("Deprecated distance [levenstein] used, replaced by [levenshtein]");
-            return new LevensteinDistance();
         } else if ("levenshtein".equals(distanceVal)) {
             return new LevensteinDistance();
-        } else if ("jarowinkler".equals(distanceVal)) {
-            DEPRECATION_LOGGER.deprecated("Deprecated distance [jarowinkler] used, replaced by [jaro_winkler]");
-            return new JaroWinklerDistance();
         } else if ("jaro_winkler".equals(distanceVal)) {
             return new JaroWinklerDistance();
         } else if ("ngram".equals(distanceVal)) {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
index dfaf77d2337..1833bb5a90e 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
@@ -581,23 +581,16 @@ public class TermSuggestionBuilder extends SuggestionBuilder DirectCandidateGeneratorBuilder.resolveDistance(null));
     }
 
-    public void testLevensteinDeprecation() {
-        assertThat(DirectCandidateGeneratorBuilder.resolveDistance("levenstein"), instanceOf(LevensteinDistance.class));
-        assertWarnings("Deprecated distance [levenstein] used, replaced by [levenshtein]");
-    }
-
-    public void testJaroWinklerDeprecation() {
-        assertThat(DirectCandidateGeneratorBuilder.resolveDistance("jaroWinkler"), instanceOf(JaroWinklerDistance.class));
-        assertWarnings("Deprecated distance [jarowinkler] used, replaced by [jaro_winkler]");
-    }
-
     private static DirectCandidateGeneratorBuilder mutate(DirectCandidateGeneratorBuilder original) throws IOException {
         DirectCandidateGeneratorBuilder mutation = copy(original);
         List> mutators = new ArrayList<>();
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/term/StringDistanceImplTests.java b/core/src/test/java/org/elasticsearch/search/suggest/term/StringDistanceImplTests.java
index ff71c469651..8ab3fadb52b 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/term/StringDistanceImplTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/term/StringDistanceImplTests.java
@@ -58,16 +58,6 @@ public class StringDistanceImplTests extends AbstractWriteableEnumTestCase {
         assertThat(e.getMessage(), equalTo("Input string is null"));
     }
 
-    public void testLevensteinDeprecation() {
-        assertThat(StringDistanceImpl.resolve("levenstein"), equalTo(StringDistanceImpl.LEVENSHTEIN));
-        assertWarnings("Deprecated distance [levenstein] used, replaced by [levenshtein]");
-    }
-
-    public void testJaroWinklerDeprecation() {
-        assertThat(StringDistanceImpl.resolve("jaroWinkler"), equalTo(StringDistanceImpl.JARO_WINKLER));
-        assertWarnings("Deprecated distance [jarowinkler] used, replaced by [jaro_winkler]");
-    }
-
     @Override
     public void testWriteTo() throws IOException {
         assertWriteToStream(StringDistanceImpl.INTERNAL, 0);
@@ -85,5 +75,4 @@ public class StringDistanceImplTests extends AbstractWriteableEnumTestCase {
         assertReadFromStream(3, StringDistanceImpl.JARO_WINKLER);
         assertReadFromStream(4, StringDistanceImpl.NGRAM);
     }
-
 }
diff --git a/docs/reference/migration/migrate_7_0/search.asciidoc b/docs/reference/migration/migrate_7_0/search.asciidoc
index 12847354cf8..aed87771b0b 100644
--- a/docs/reference/migration/migrate_7_0/search.asciidoc
+++ b/docs/reference/migration/migrate_7_0/search.asciidoc
@@ -35,8 +35,17 @@ The Search API returns `400 - Bad request` while it would previously return
 *   number of filters in the adjacency matrix aggregation is too large
 
 
-==== Scroll queries cannot use the request_cache anymore
+==== Scroll queries cannot use the `request_cache` anymore
 
-Setting `request_cache:true` on a query that creates a scroll ('scroll=1m`)
+Setting `request_cache:true` on a query that creates a scroll (`scroll=1m`)
 has been deprecated in 6 and will now return a `400 - Bad request`.
 Scroll queries are not meant to be cached.
+
+==== Term Suggesters supported distance algorithms
+
+The following string distance algorithms were given additional names in 6.2 and
+their existing names were deprecated. The deprecated names have now been
+removed.
+
+* 	`levenstein` - replaced by `levenshtein`
+* 	`jarowinkler` - replaced by `jaro_winkler`

From b35c459c96d806fc518618eec663c2890b7e90fb Mon Sep 17 00:00:00 2001
From: Jim Ferenczi 
Date: Mon, 11 Dec 2017 13:31:58 +0100
Subject: [PATCH 235/297] [TESTS] Fix expectations for
 GeoShapeQueryBuilderTests#testWrongFieldType

Relates #27730
---
 .../org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
index 8c08725aebb..6ddcd671a40 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
@@ -260,6 +260,7 @@ public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase 0);
         ShapeType shapeType = ShapeType.randomType(random());
         ShapeBuilder shape = RandomShapeGenerator.createShapeWithin(random(), null, shapeType);
         final GeoShapeQueryBuilder queryBuilder = new GeoShapeQueryBuilder(STRING_FIELD_NAME, shape);

From e01643126b976b3f6345b1a91810f0a50b45408e Mon Sep 17 00:00:00 2001
From: javanna 
Date: Mon, 11 Dec 2017 14:27:01 +0100
Subject: [PATCH 236/297] [TEST] extend wait_for_active_shards randomization to
 include 'all' value

This was already changed in 6.x as part of the backport of the recently added open and create index API. wait_for_active_shards can be a number but also "all", with this commit we verify that providing "all" works too.
---
 .../org/elasticsearch/client/RequestTests.java    | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
index 7d63ed1ed40..019162bae37 100755
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java
@@ -38,6 +38,7 @@ import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -1019,11 +1020,17 @@ public class RequestTests extends ESTestCase {
         }
     }
 
-    private static void setRandomWaitForActiveShards(Consumer setter, Map expectedParams) {
+    private static void setRandomWaitForActiveShards(Consumer setter, Map expectedParams) {
         if (randomBoolean()) {
-            int waitForActiveShards = randomIntBetween(0, 10);
-            setter.accept(waitForActiveShards);
-            expectedParams.put("wait_for_active_shards", String.valueOf(waitForActiveShards));
+            String waitForActiveShardsString;
+            int waitForActiveShards = randomIntBetween(-1, 5);
+            if (waitForActiveShards == -1) {
+                waitForActiveShardsString = "all";
+            } else {
+                waitForActiveShardsString = String.valueOf(waitForActiveShards);
+            }
+            setter.accept(ActiveShardCount.parseString(waitForActiveShardsString));
+            expectedParams.put("wait_for_active_shards", waitForActiveShardsString);
         }
     }
 

From 22e294ce6d9034e715b6f166546d39ec827eb2c7 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Mon, 11 Dec 2017 10:18:06 -0500
Subject: [PATCH 237/297] Fix performance of RoutingNodes#assertShardStats

The performance of this method is abysmal, it leads to the
balanced/unbalanced cluster tests taking twenty seconds! The reason for
the performance issue is a quadruple-nested for loop. The inner
double-nested loop is partitioning shards by shard ID in disguise, so we
simply extract this into computing a partition of shards by shard ID
once. Now balanced/unbalanced cluster test does not take twenty seconds
to run.

Relates #27747
---
 .../cluster/routing/RoutingNodes.java         | 36 ++++++++++---------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index c5f0cb82feb..43711525bcf 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -40,6 +40,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -1043,26 +1044,27 @@ public class RoutingNodes implements Iterable {
                 indicesAndShards.put(shard.index(), Math.max(i, shard.id()));
             }
         }
+
         // Assert that the active shard routing are identical.
         Set> entries = indicesAndShards.entrySet();
-        final List shards = new ArrayList<>();
-        for (Map.Entry e : entries) {
-            Index index = e.getKey();
+
+        final Map> shardsByShardId = new HashMap<>();
+        for (final RoutingNode routingNode: routingNodes) {
+            for (final ShardRouting shardRouting : routingNode) {
+                final HashSet shards =
+                        shardsByShardId.computeIfAbsent(new ShardId(shardRouting.index(), shardRouting.id()), k -> new HashSet<>());
+                shards.add(shardRouting);
+            }
+        }
+
+        for (final Map.Entry e : entries) {
+            final Index index = e.getKey();
             for (int i = 0; i < e.getValue(); i++) {
-                for (RoutingNode routingNode : routingNodes) {
-                    for (ShardRouting shardRouting : routingNode) {
-                        if (shardRouting.index().equals(index) && shardRouting.id() == i) {
-                            shards.add(shardRouting);
-                        }
-                    }
-                }
-                List mutableShardRoutings = routingNodes.assignedShards(new ShardId(index, i));
-                assert mutableShardRoutings.size() == shards.size();
-                for (ShardRouting r : mutableShardRoutings) {
-                    assert shards.contains(r);
-                    shards.remove(r);
-                }
-                assert shards.isEmpty();
+                final ShardId shardId = new ShardId(index, i);
+                final HashSet shards = shardsByShardId.get(shardId);
+                final List mutableShardRoutings = routingNodes.assignedShards(shardId);
+                assert (shards == null && mutableShardRoutings.size() == 0)
+                        || (shards != null && shards.size() == mutableShardRoutings.size() && shards.containsAll(mutableShardRoutings));
             }
         }
 

From ebb93db01029698d0204991a9e75be791f9c6306 Mon Sep 17 00:00:00 2001
From: Simon Willnauer 
Date: Mon, 11 Dec 2017 16:39:06 +0100
Subject: [PATCH 238/297] Remove pre 6.0.0 support from InternalEngine (#27720)

This removes special casing for documents without a sequence ID.
This code is complex enough with seq IDs we should clean up things
when we can and we don't support 5.x indexing in 7.x anymore
---
 .../index/engine/InternalEngine.java          | 33 +++----------------
 .../index/engine/InternalEngineTests.java     | 18 ----------
 2 files changed, 5 insertions(+), 46 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 53747b063df..ecf589cbaf9 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -700,13 +700,9 @@ public class InternalEngine extends Engine {
     }
 
     private boolean assertIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) {
-        if (engineConfig.getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1) && origin == Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
-            // legacy support
-            assert seqNo == SequenceNumbers.UNASSIGNED_SEQ_NO : "old op recovering but it already has a seq no.;" +
-                " index version: " + engineConfig.getIndexSettings().getIndexVersionCreated() + ", seqNo: " + seqNo;
-        } else if (origin == Operation.Origin.PRIMARY) {
+        if (origin == Operation.Origin.PRIMARY) {
             assert assertOriginPrimarySequenceNumber(seqNo);
-        } else if (engineConfig.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1)) {
+        } else {
             // sequence number should be set when operation origin is not primary
             assert seqNo >= 0 : "recovery or replica ops should have an assigned seq no.; origin: " + origin;
         }
@@ -720,15 +716,6 @@ public class InternalEngine extends Engine {
         return true;
     }
 
-    private boolean assertSequenceNumberBeforeIndexing(final Engine.Operation.Origin origin, final long seqNo) {
-        if (engineConfig.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1) ||
-            origin == Operation.Origin.PRIMARY) {
-            // sequence number should be set when operation origin is primary or when all shards are on new nodes
-            assert seqNo >= 0 : "ops should have an assigned seq no.; origin: " + origin;
-        }
-        return true;
-    }
-
     private long generateSeqNoForOperation(final Operation operation) {
         assert operation.origin() == Operation.Origin.PRIMARY;
         return doGenerateSeqNoForOperation(operation);
@@ -842,13 +829,7 @@ public class InternalEngine extends Engine {
             // this allows to ignore the case where a document was found in the live version maps in
             // a delete state and return false for the created flag in favor of code simplicity
             final OpVsLuceneDocStatus opVsLucene;
-            if (index.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO) {
-                // This can happen if the primary is still on an old node and send traffic without seq# or we recover from translog
-                // created by an old version.
-                assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1) :
-                    "index is newly created but op has no sequence numbers. op: " + index;
-                opVsLucene = compareOpToLuceneDocBasedOnVersions(index);
-            } else if (index.seqNo() <= seqNoService.getLocalCheckpoint()){
+            if (index.seqNo() <= seqNoService.getLocalCheckpoint()){
                 // the operation seq# is lower then the current local checkpoint and thus was already put into lucene
                 // this can happen during recovery where older operations are sent from the translog that are already
                 // part of the lucene commit (either from a peer recovery or a local translog)
@@ -910,7 +891,7 @@ public class InternalEngine extends Engine {
 
     private IndexResult indexIntoLucene(Index index, IndexingStrategy plan)
         throws IOException {
-        assert assertSequenceNumberBeforeIndexing(index.origin(), plan.seqNoForIndexing);
+        assert plan.seqNoForIndexing >= 0 : "ops should have an assigned seq no.; origin: " + index.origin();
         assert plan.versionForIndexing >= 0 : "version must be set. got " + plan.versionForIndexing;
         assert plan.indexIntoLucene;
         /* Update the document's sequence number and primary term; the sequence number here is derived here from either the sequence
@@ -1135,11 +1116,7 @@ public class InternalEngine extends Engine {
         // this allows to ignore the case where a document was found in the live version maps in
         // a delete state and return true for the found flag in favor of code simplicity
         final OpVsLuceneDocStatus opVsLucene;
-        if (delete.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO) {
-            assert config().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1) :
-                "index is newly created but op has no sequence numbers. op: " + delete;
-            opVsLucene = compareOpToLuceneDocBasedOnVersions(delete);
-        } else if (delete.seqNo() <= seqNoService.getLocalCheckpoint()) {
+        if (delete.seqNo() <= seqNoService.getLocalCheckpoint()) {
             // the operation seq# is lower then the current local checkpoint and thus was already put into lucene
             // this can happen during recovery where older operations are sent from the translog that are already
             // part of the lucene commit (either from a peer recovery or a local translog)
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 8a133f35c24..2d772daf7cc 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -1353,24 +1353,6 @@ public class InternalEngineTests extends EngineTestCase {
         assertOpsOnReplica(ops, replicaEngine, true);
     }
 
-    public void testOutOfOrderDocsOnReplicaOldPrimary() throws IOException {
-        IndexSettings oldSettings = IndexSettingsModule.newIndexSettings("testOld", Settings.builder()
-            .put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), "1h") // make sure this doesn't kick in on us
-            .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codecName)
-            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_5_4_0)
-            .put(IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, true)
-            .put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.getKey(),
-                between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY)))
-            .build());
-
-        try (Store oldReplicaStore = createStore();
-             InternalEngine replicaEngine =
-                 createEngine(oldSettings, oldReplicaStore, createTempDir("translog-old-replica"), newMergePolicy())) {
-            final List ops = generateSingleDocHistory(true, randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), true, 2, 2, 20);
-            assertOpsOnReplica(ops, replicaEngine, true);
-        }
-    }
-
     private void assertOpsOnReplica(List ops, InternalEngine replicaEngine, boolean shuffleOps) throws IOException {
         final Engine.Operation lastOp = ops.get(ops.size() - 1);
         final String lastFieldValue;

From 1cd55759971faaf506bf46803701dfa2bbecd27a Mon Sep 17 00:00:00 2001
From: Andrew Banchich 
Date: Mon, 11 Dec 2017 12:05:39 -0500
Subject: [PATCH 239/297] Update query-dsl.asciidoc (#27669)

---
 docs/reference/query-dsl.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/query-dsl.asciidoc b/docs/reference/query-dsl.asciidoc
index 1eb6fbddd1e..8d9c803b615 100644
--- a/docs/reference/query-dsl.asciidoc
+++ b/docs/reference/query-dsl.asciidoc
@@ -4,8 +4,8 @@
 [partintro]
 --
 
-Elasticsearch provides a full Query DSL based on JSON to define queries.
-Think of the Query DSL as an AST of queries, consisting of two types of
+Elasticsearch provides a full Query DSL (Domain Specific Language) based on JSON to define queries.
+Think of the Query DSL as an AST (Abstract Syntax Tree) of queries, consisting of two types of
 clauses:
 
 Leaf query clauses::

From 6bc40e4bd389451aaec0e5900b43733439fb1dba Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Mon, 11 Dec 2017 13:26:27 -0500
Subject: [PATCH 240/297] No longer unidle shard during recovery

Previously we would unidle a primary shard during recovery in case the
recovery target would miss a background global checkpoint sync. However,
the background global checkpoint syncs are no longer tied to the primary
shard falling idle and so this unidling is no longer needed.

Relates #27757
---
 .../main/java/org/elasticsearch/index/shard/IndexShard.java  | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index f0246060acf..b3847cd1b4d 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1745,11 +1745,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     public void initiateTracking(final String allocationId) {
         verifyPrimary();
         getEngine().seqNoService().initiateTracking(allocationId);
-        /*
-         * We could have blocked so long waiting for the replica to catch up that we fell idle and there will not be a background sync to
-         * the replica; mark our self as active to force a future background sync.
-         */
-        active.compareAndSet(false, true);
     }
 
     /**

From cd474df9725d63fb2b76481f44f3f26850c8768b Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Mon, 11 Dec 2017 15:40:10 -0500
Subject: [PATCH 241/297] Remove RPM and Debian integration tests

We have tests that manually unpackage the RPM and Debian package
distributions and start a cluster manually (not from the service) and
run a basic suite of integration tests against them. This is problematic
because it is not how the packages are intended to be used (instead,
they are intended to be installed using the package installation tools,
and started as services) and so violates assumptions that we make about
directory paths. This commit removes these integration tests, instead
relying on the packaging tests to ensure the packages are not
broken. Additionally, we add a sanity check that the package
distributions can be unpackaged. Finally, with this change we can remove
some leniency from elasticsearch-env about checking for the existence of
the environment file which the leniency was there solely for these
integration tests.

Relates #27725
---
 .../gradle/test/ClusterFormationTasks.groovy  | 29 ---------------
 distribution/build.gradle                     |  4 +-
 distribution/deb/build.gradle                 | 21 ++++++++---
 .../test/rest/DebClientYamlTestSuiteIT.java   | 36 ------------------
 .../test/smoke_test_plugins/10_modules.yml    | 13 -------
 distribution/rpm/build.gradle                 | 34 ++++++++++++++---
 .../test/rest/RpmClientYamlTestSuiteIT.java   | 37 -------------------
 .../test/smoke_test_plugins/10_modules.yml    | 13 -------
 8 files changed, 46 insertions(+), 141 deletions(-)
 delete mode 100644 distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
 delete mode 100644 distribution/deb/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml
 delete mode 100644 distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
 delete mode 100644 distribution/rpm/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index f7637d23366..2b3b5abd82c 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -269,35 +269,6 @@ class ClusterFormationTasks {
                     into node.baseDir
                 }
                 break;
-            case 'rpm':
-                File rpmDatabase = new File(node.baseDir, 'rpm-database')
-                File rpmExtracted = new File(node.baseDir, 'rpm-extracted')
-                /* Delay reading the location of the rpm file until task execution */
-                Object rpm = "${ -> configuration.singleFile}"
-                extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) {
-                    commandLine 'rpm', '--badreloc', '--nodeps', '--noscripts', '--notriggers',
-                        '--dbpath', rpmDatabase,
-                        '--relocate', "/=${rpmExtracted}",
-                        '-i', rpm
-                    doFirst {
-                        rpmDatabase.deleteDir()
-                        rpmExtracted.deleteDir()
-                    }
-                    outputs.dir rpmExtracted
-                }
-                break;
-            case 'deb':
-                /* Delay reading the location of the deb file until task execution */
-                File debExtracted = new File(node.baseDir, 'deb-extracted')
-                Object deb = "${ -> configuration.singleFile}"
-                extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) {
-                    commandLine 'dpkg-deb', '-x', deb, debExtracted
-                    doFirst {
-                        debExtracted.deleteDir()
-                    }
-                    outputs.dir debExtracted
-                }
-                break;
             default:
                 throw new InvalidUserDataException("Unknown distribution: ${node.config.distribution}")
         }
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 3df81d24c6b..26221cad5c7 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -538,8 +538,8 @@ Map expansionsForDistribution(distributionType) {
       'def': 'if [ -z "$ES_PATH_CONF" ]; then ES_PATH_CONF="$ES_HOME"/config; done',
     ],
     'source.path.env': [
-       'deb': 'if [ -f /etc/default/elasticsearch ]; then source /etc/default/elasticsearch; fi',
-       'rpm': 'if [ -f /etc/sysconfig/elasticsearch ]; then source /etc/sysconfig/elasticsearch; fi',
+       'deb': 'source /etc/default/elasticsearch',
+       'rpm': 'source /etc/sysconfig/elasticsearch',
        'def': 'if [ -z "$ES_PATH_CONF" ]; then ES_PATH_CONF="$ES_HOME"/config; fi',
     ],
     'path.logs': [
diff --git a/distribution/deb/build.gradle b/distribution/deb/build.gradle
index 89d33597888..cfc9aa4d8ef 100644
--- a/distribution/deb/build.gradle
+++ b/distribution/deb/build.gradle
@@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.LoggedExec
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -42,9 +44,18 @@ artifacts {
   archives buildDeb
 }
 
-integTest {
-  /* We use real deb tools to extract the deb file for testing so we have to
-    skip the test if they aren't around. */
-  enabled = new File('/usr/bin/dpkg-deb').exists() || // Standard location
-      new File('/usr/local/bin/dpkg-deb').exists()    // Homebrew location
+integTest.enabled = false
+licenseHeaders.enabled = false
+
+// task that sanity checks if the Deb archive can be extracted
+task checkDeb(type: LoggedExec) {
+  onlyIf { new File('/usr/bin/dpkg-deb').exists() || new File('/usr/local/bin/dpkg-deb').exists() }
+  final File debExtracted = new File("${buildDir}", 'deb-extracted')
+  commandLine 'dpkg-deb', '-x', "${buildDir}/distributions/elasticsearch-${project.version}.deb", debExtracted
+  doFirst {
+    debExtracted.deleteDir()
+  }
 }
+
+checkDeb.dependsOn buildDeb
+check.dependsOn checkDeb
diff --git a/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java b/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
deleted file mode 100644
index 0fcdecde068..00000000000
--- a/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest;
-
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-
-/** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
-public class DebClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
-    public DebClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    @ParametersFactory
-    public static Iterable parameters() throws Exception {
-        return createParameters();
-    }
-}
diff --git a/distribution/deb/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml b/distribution/deb/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml
deleted file mode 100644
index da68232f8d8..00000000000
--- a/distribution/deb/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-# Integration tests for distributions with modules
-#
-"Correct Modules Count":
-    - do:
-        cluster.state: {}
-
-    # Get master node id
-    - set: { master_node: master }
-
-    - do:
-        nodes.info: {}
-
-    - length:  { nodes.$master.modules: ${expected.modules.count}  }
diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle
index 6f8299522ca..4432198f234 100644
--- a/distribution/rpm/build.gradle
+++ b/distribution/rpm/build.gradle
@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import org.elasticsearch.gradle.LoggedExec
+
 task buildRpm(type: Rpm) {
   dependsOn preparePackagingFiles
   baseName 'elasticsearch' // this is what pom generation uses for artifactId
@@ -54,10 +56,30 @@ artifacts {
   archives buildRpm
 }
 
-integTest {
-  /* We use real rpm tools to extract the rpm file for testing so we have to
-    skip the test if they aren't around. */
-  enabled = new File('/bin/rpm').exists() ||   // Standard location
-      new File('/usr/bin/rpm').exists() ||     // Debian location
-      new File('/usr/local/bin/rpm').exists()  // Homebrew location
+integTest.enabled = false
+licenseHeaders.enabled = false
+
+// task that sanity checks if the RPM archive can be extracted
+task checkRpm(type: LoggedExec) {
+  onlyIf { new File('/bin/rpm').exists() || new File('/usr/bin/rpm').exists() || new File('/usr/local/bin/rpm').exists() }
+  final File rpmDatabase = new File("${buildDir}", 'rpm-database')
+  final File rpmExtracted = new File("${buildDir}", 'rpm-extracted')
+  commandLine 'rpm',
+          '--badreloc',
+          '--nodeps',
+          '--noscripts',
+          '--notriggers',
+          '--dbpath',
+          rpmDatabase,
+          '--relocate',
+          "/=${rpmExtracted}",
+          '-i',
+          "${buildDir}/distributions/elasticsearch-${project.version}.rpm"
+  doFirst {
+    rpmDatabase.deleteDir()
+    rpmExtracted.deleteDir()
+  }
 }
+
+checkRpm.dependsOn buildRpm
+check.dependsOn checkRpm
diff --git a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
deleted file mode 100644
index 388f5ee6517..00000000000
--- a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest;
-
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
-import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-
-/** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
-public class RpmClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
-    public RpmClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    @ParametersFactory
-    public static Iterable parameters() throws Exception {
-        return createParameters();
-    }
-}
diff --git a/distribution/rpm/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml b/distribution/rpm/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml
deleted file mode 100644
index da68232f8d8..00000000000
--- a/distribution/rpm/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-# Integration tests for distributions with modules
-#
-"Correct Modules Count":
-    - do:
-        cluster.state: {}
-
-    # Get master node id
-    - set: { master_node: master }
-
-    - do:
-        nodes.info: {}
-
-    - length:  { nodes.$master.modules: ${expected.modules.count}  }

From 8188d9f7e50acd576c30a66ed6ca845d9081f277 Mon Sep 17 00:00:00 2001
From: Jack Conradson 
Date: Mon, 11 Dec 2017 16:37:35 -0800
Subject: [PATCH 242/297] Painless: Only allow Painless type names to be the
 same as the equivalent Java class. (#27264)

Also adds a parameter called only_fqn to the whitelist to enforce that a painless type must be specified as the fully-qualifed java class name.
---
 docs/painless/painless-debugging.asciidoc     |   2 +-
 .../elasticsearch/painless/Definition.java    |  94 ++++++--
 .../org/elasticsearch/painless/Whitelist.java |  12 +-
 .../painless/WhitelistLoader.java             |  56 +++--
 .../org/elasticsearch/painless/java.lang.txt  | 106 ++++-----
 .../org/elasticsearch/painless/java.math.txt  |   8 +-
 .../org/elasticsearch/painless/java.text.txt  |  60 ++---
 .../painless/java.time.chrono.txt             |  42 ++--
 .../painless/java.time.format.txt             |  16 +-
 .../painless/java.time.temporal.txt           |  32 +--
 .../org/elasticsearch/painless/java.time.txt  |  36 +--
 .../elasticsearch/painless/java.time.zone.txt |  12 +-
 .../painless/java.util.function.txt           |  86 +++----
 .../painless/java.util.regex.txt              |   4 +-
 .../painless/java.util.stream.txt             |  24 +-
 .../org/elasticsearch/painless/java.util.txt  | 216 +++++++++---------
 .../org/elasticsearch/painless/joda.time.txt  |   4 +-
 .../painless/org.elasticsearch.txt            |  52 ++---
 .../elasticsearch/painless/DebugTests.java    |   6 +-
 .../elasticsearch/painless/RegexTests.java    |   6 +-
 .../painless/node/NodeToStringTests.java      |   5 +-
 21 files changed, 474 insertions(+), 405 deletions(-)

diff --git a/docs/painless/painless-debugging.asciidoc b/docs/painless/painless-debugging.asciidoc
index e50090fe71d..a909593ff17 100644
--- a/docs/painless/painless-debugging.asciidoc
+++ b/docs/painless/painless-debugging.asciidoc
@@ -78,7 +78,7 @@ The response looks like:
     "caused_by": {
       "type": "script_exception",
       "to_string": "{gp=[26, 82, 1], last=gaudreau, assists=[17, 46, 0], first=johnny, goals=[9, 27, 1]}",
-      "painless_class": "LinkedHashMap",
+      "painless_class": "java.util.LinkedHashMap",
       "java_class": "java.util.LinkedHashMap",
       ...
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index df56c599f03..43de306a7a8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -33,6 +33,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Stack;
+import java.util.regex.Pattern;
 
 /**
  * The entire API for Painless.  Also used as a whitelist for checking for legal
@@ -40,6 +41,8 @@ import java.util.Stack;
  */
 public final class Definition {
 
+    private static final Pattern TYPE_NAME_PATTERN = Pattern.compile("^[_a-zA-Z][._a-zA-Z0-9]*$");
+
     private static final String[] DEFINITION_FILES = new String[] {
             "org.elasticsearch.txt",
             "java.lang.txt",
@@ -535,7 +538,8 @@ public final class Definition {
             // are used for validation during the second iteration
             for (Whitelist whitelist : whitelists) {
                 for (Whitelist.Struct whitelistStruct : whitelist.whitelistStructs) {
-                    Struct painlessStruct = structsMap.get(whitelistStruct.painlessTypeName);
+                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
+                    Struct painlessStruct = structsMap.get(painlessTypeName);
 
                     if (painlessStruct != null && painlessStruct.clazz.getName().equals(whitelistStruct.javaClassName) == false) {
                         throw new IllegalArgumentException("struct [" + painlessStruct.name + "] cannot represent multiple classes " +
@@ -545,7 +549,7 @@ public final class Definition {
                     origin = whitelistStruct.origin;
                     addStruct(whitelist.javaClassLoader, whitelistStruct);
 
-                    painlessStruct = structsMap.get(whitelistStruct.painlessTypeName);
+                    painlessStruct = structsMap.get(painlessTypeName);
                     javaClassesToPainlessStructs.put(painlessStruct.clazz, painlessStruct);
                 }
             }
@@ -555,19 +559,21 @@ public final class Definition {
             // been white-listed during the first iteration
             for (Whitelist whitelist : whitelists) {
                 for (Whitelist.Struct whitelistStruct : whitelist.whitelistStructs) {
+                    String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
+
                     for (Whitelist.Constructor whitelistConstructor : whitelistStruct.whitelistConstructors) {
                         origin = whitelistConstructor.origin;
-                        addConstructor(whitelistStruct.painlessTypeName, whitelistConstructor);
+                        addConstructor(painlessTypeName, whitelistConstructor);
                     }
 
                     for (Whitelist.Method whitelistMethod : whitelistStruct.whitelistMethods) {
                         origin = whitelistMethod.origin;
-                        addMethod(whitelist.javaClassLoader, whitelistStruct.painlessTypeName, whitelistMethod);
+                        addMethod(whitelist.javaClassLoader, painlessTypeName, whitelistMethod);
                     }
 
                     for (Whitelist.Field whitelistField : whitelistStruct.whitelistFields) {
                         origin = whitelistField.origin;
-                        addField(whitelistStruct.painlessTypeName, whitelistField);
+                        addField(painlessTypeName, whitelistField);
                     }
                 }
             }
@@ -577,7 +583,14 @@ public final class Definition {
 
         // goes through each Painless struct and determines the inheritance list,
         // and then adds all inherited types to the Painless struct's whitelist
-        for (Struct painlessStruct : structsMap.values()) {
+        for (Map.Entry painlessNameStructEntry : structsMap.entrySet()) {
+            String painlessStructName = painlessNameStructEntry.getKey();
+            Struct painlessStruct = painlessNameStructEntry.getValue();
+
+            if (painlessStruct.name.equals(painlessStructName) == false) {
+                continue;
+            }
+
             List painlessSuperStructs = new ArrayList<>();
             Class javaSuperClass = painlessStruct.clazz.getSuperclass();
 
@@ -633,16 +646,33 @@ public final class Definition {
         }
 
         // mark functional interfaces (or set null, to mark class is not)
-        for (Struct clazz : structsMap.values()) {
-            clazz.functionalMethod.set(computeFunctionalInterfaceMethod(clazz));
+        for (String painlessStructName : structsMap.keySet()) {
+            Struct painlessStruct = structsMap.get(painlessStructName);
+
+            if (painlessStruct.name.equals(painlessStructName) == false) {
+                continue;
+            }
+
+            painlessStruct.functionalMethod.set(computeFunctionalInterfaceMethod(painlessStruct));
         }
 
         // precompute runtime classes
-        for (Struct struct : structsMap.values()) {
-            addRuntimeClass(struct);
+        for (String painlessStructName : structsMap.keySet()) {
+            Struct painlessStruct = structsMap.get(painlessStructName);
+
+            if (painlessStruct.name.equals(painlessStructName) == false) {
+                continue;
+            }
+
+            addRuntimeClass(painlessStruct);
         }
+
         // copy all structs to make them unmodifiable for outside users:
-        for (final Map.Entry entry : structsMap.entrySet()) {
+        for (Map.Entry entry : structsMap.entrySet()) {
+            if (entry.getKey().equals(entry.getValue().name) == false) {
+                continue;
+            }
+
             entry.setValue(entry.getValue().freeze());
         }
 
@@ -678,8 +708,17 @@ public final class Definition {
     }
 
     private void addStruct(ClassLoader whitelistClassLoader, Whitelist.Struct whitelistStruct) {
-        if (!whitelistStruct.painlessTypeName.matches("^[_a-zA-Z][._a-zA-Z0-9]*")) {
-            throw new IllegalArgumentException("invalid struct type name [" + whitelistStruct.painlessTypeName + "]");
+        String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.');
+        String importedPainlessTypeName = painlessTypeName;
+
+        if (TYPE_NAME_PATTERN.matcher(painlessTypeName).matches() == false) {
+            throw new IllegalArgumentException("invalid struct type name [" + painlessTypeName + "]");
+        }
+
+        int index = whitelistStruct.javaClassName.lastIndexOf('.');
+
+        if (index != -1) {
+            importedPainlessTypeName = whitelistStruct.javaClassName.substring(index + 1).replace('$', '.');
         }
 
         Class javaClass;
@@ -698,21 +737,34 @@ public final class Definition {
                 javaClass = Class.forName(whitelistStruct.javaClassName, true, whitelistClassLoader);
             } catch (ClassNotFoundException cnfe) {
                 throw new IllegalArgumentException("invalid java class name [" + whitelistStruct.javaClassName + "]" +
-                        " for struct [" + whitelistStruct.painlessTypeName + "]");
+                        " for struct [" + painlessTypeName + "]");
             }
         }
 
-        Struct existingStruct = structsMap.get(whitelistStruct.painlessTypeName);
+        Struct existingStruct = structsMap.get(painlessTypeName);
 
         if (existingStruct == null) {
-            Struct struct = new Struct(whitelistStruct.painlessTypeName, javaClass, org.objectweb.asm.Type.getType(javaClass));
+            Struct struct = new Struct(painlessTypeName, javaClass, org.objectweb.asm.Type.getType(javaClass));
+            structsMap.put(painlessTypeName, struct);
 
-            structsMap.put(whitelistStruct.painlessTypeName, struct);
-            simpleTypesMap.put(whitelistStruct.painlessTypeName, getTypeInternal(whitelistStruct.painlessTypeName));
+            if (whitelistStruct.onlyFQNJavaClassName) {
+                simpleTypesMap.put(painlessTypeName, getType(painlessTypeName));
+            } else if (simpleTypesMap.containsKey(importedPainlessTypeName) == false) {
+                simpleTypesMap.put(importedPainlessTypeName, getType(painlessTypeName));
+                structsMap.put(importedPainlessTypeName, struct);
+            } else {
+                throw new IllegalArgumentException("duplicate short name [" + importedPainlessTypeName + "] " +
+                        "found for struct [" + painlessTypeName + "]");
+            }
         } else if (existingStruct.clazz.equals(javaClass) == false) {
-            throw new IllegalArgumentException("struct [" + whitelistStruct.painlessTypeName + "] is used to " +
+            throw new IllegalArgumentException("struct [" + painlessTypeName + "] is used to " +
                     "illegally represent multiple java classes [" + whitelistStruct.javaClassName + "] and " +
                     "[" + existingStruct.clazz.getName() + "]");
+        } else if (whitelistStruct.onlyFQNJavaClassName && simpleTypesMap.containsKey(importedPainlessTypeName) &&
+                simpleTypesMap.get(importedPainlessTypeName).clazz == javaClass ||
+                whitelistStruct.onlyFQNJavaClassName == false && (simpleTypesMap.containsKey(importedPainlessTypeName) == false ||
+                simpleTypesMap.get(importedPainlessTypeName).clazz != javaClass)) {
+            throw new IllegalArgumentException("inconsistent only_fqn parameters found for type [" + painlessTypeName + "]");
         }
     }
 
@@ -783,7 +835,7 @@ public final class Definition {
                     "name [" + whitelistMethod.javaMethodName + "] and parameters " + whitelistMethod.painlessParameterTypeNames);
         }
 
-        if (!whitelistMethod.javaMethodName.matches("^[_a-zA-Z][_a-zA-Z0-9]*$")) {
+        if (TYPE_NAME_PATTERN.matcher(whitelistMethod.javaMethodName).matches() == false) {
             throw new IllegalArgumentException("invalid method name" +
                     " [" + whitelistMethod.javaMethodName + "] for owner struct [" + ownerStructName + "].");
         }
@@ -913,7 +965,7 @@ public final class Definition {
                     "name [" + whitelistField.javaFieldName + "] and type " + whitelistField.painlessFieldTypeName);
         }
 
-        if (!whitelistField.javaFieldName.matches("^[_a-zA-Z][_a-zA-Z0-9]*$")) {
+        if (TYPE_NAME_PATTERN.matcher(whitelistField.javaFieldName).matches() == false) {
             throw new IllegalArgumentException("invalid field name " +
                     "[" + whitelistField.painlessFieldTypeName + "] for owner struct [" + ownerStructName + "].");
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Whitelist.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Whitelist.java
index 7fd3493d517..678b8a4c1ae 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Whitelist.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Whitelist.java
@@ -56,12 +56,14 @@ public final class Whitelist {
         /** Information about where this struct was white-listed from.  Can be used for error messages. */
         public final String origin;
 
-        /** The Painless name of this struct which will also be the name of a type in a Painless script.  */
-        public final String painlessTypeName;
-
         /** The Java class name this struct represents. */
         public final String javaClassName;
 
+        /**
+         * Allow the Java class name to only be specified as the fully-qualified name.
+         */
+        public final boolean onlyFQNJavaClassName;
+
         /** The {@link List} of white-listed ({@link Constructor}s) available to this struct. */
         public final List whitelistConstructors;
 
@@ -72,11 +74,11 @@ public final class Whitelist {
         public final List whitelistFields;
 
         /** Standard constructor. All values must be not {@code null}. */
-        public Struct(String origin, String painlessTypeName, String javaClassName,
+        public Struct(String origin, String javaClassName, boolean onlyFQNJavaClassName,
                       List whitelistConstructors, List whitelistMethods, List whitelistFields) {
             this.origin = Objects.requireNonNull(origin);
-            this.painlessTypeName = Objects.requireNonNull(painlessTypeName);
             this.javaClassName = Objects.requireNonNull(javaClassName);
+            this.onlyFQNJavaClassName = onlyFQNJavaClassName;
 
             this.whitelistConstructors = Collections.unmodifiableList(Objects.requireNonNull(whitelistConstructors));
             this.whitelistMethods = Collections.unmodifiableList(Objects.requireNonNull(whitelistMethods));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WhitelistLoader.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WhitelistLoader.java
index ad33d9c7ba5..93ea951f453 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WhitelistLoader.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WhitelistLoader.java
@@ -43,18 +43,29 @@ public final class WhitelistLoader {
      * and field.  Most validation will be done at a later point after all white-lists have been gathered and their
      * merging takes place.
      *
+     * A painless type name is one of the following:
+     * 
      
+     *     
    •  def - The Painless dynamic type which is automatically included without a need to be
+     *     white-listed. 
      • 
+     *     
    •  fully-qualified Java type name - Any white-listed Java class will have the equivalent name as
+     *     a Painless type name with the exception that any dollar symbols used as part of inner classes will
+     *     be replaced with dot symbols. 
      • 
+     *     
    •  short Java type name - The text after the final dot symbol of any specified Java class.  A
+     *     short type Java name may be excluded by using the 'only_fqn' token during Painless struct parsing
+     *     as described later. 
      • 
+     * 
    
+     *
      * The following can be parsed from each white-list text file:
      * 
      
      *   
    •  Blank lines will be ignored by the parser. 
      • 
      *   
    •  Comments may be created starting with a pound '#' symbol and end with a newline.  These will
      *   be ignored by the parser. 
      • 
-     *   
    •  Primitive types may be specified starting with 'class' and followed by the Painless type
-     *   name (often the same as the Java type name), an arrow symbol, the Java type name,
+     *   
    •  Primitive types may be specified starting with 'class' and followed by the Java type name,
      *   an opening bracket, a newline, a closing bracket, and a final newline. 
      • 
-     *   
    •  Complex types may be specified starting with 'class' and followed by the Painless type name,
-     *   an arrow symbol, the Java class name, a opening bracket, a newline, constructor/method/field
-     *   specifications, a closing bracket, and a final newline. Within a complex type the following
-     *   may be parsed:
+     *   
    •  Complex types may be specified starting with 'class' and followed the fully-qualified Java
+     *   class name, optionally followed by an 'only_fqn' token, an opening bracket, a newline,
+     *   constructor/method/field specifications, a closing bracket, and a final newline. Within a complex
+     *   type the following may be parsed:
      *   
        
      *     
      •  A constructor may be specified starting with an opening parenthesis, followed by a
      *     comma-delimited list of Painless type names corresponding to the type/class names for
@@ -82,7 +93,9 @@ public final class WhitelistLoader {
      * If the same Painless type is defined across multiple files and the Java class is the same, all
      * specified constructors, methods, and fields will be merged into a single Painless type.  The
      * Painless dynamic type, 'def', used as part of constructor, method, and field definitions will
-     * be appropriately parsed and handled.
+     * be appropriately parsed and handled.  Painless complex types must be specified with the
+     * fully-qualified Java class name.  Method argument types, method return types, and field types
+     * must be specified with Painless type names (def, fully-qualified, or short) as described earlier.
      *
      * The following example is used to create a single white-list text file:
      *
@@ -94,7 +107,7 @@ public final class WhitelistLoader {
      *
      * # complex types
      *
-     * class Example -> my.package.Example {
+     * class my.package.Example only_fqn {
      *   # constructors
      *   ()
      *   (int)
@@ -129,8 +142,8 @@ public final class WhitelistLoader {
                 new InputStreamReader(resource.getResourceAsStream(filepath), StandardCharsets.UTF_8))) {
 
                 String whitelistStructOrigin = null;
-                String painlessTypeName = null;
                 String javaClassName = null;
+                boolean onlyFQNJavaClassName = false;
                 List whitelistConstructors = null;
                 List whitelistMethods = null;
                 List whitelistFields = null;
@@ -145,7 +158,7 @@ public final class WhitelistLoader {
                     }
 
                     // Handle a new struct by resetting all the variables necessary to construct a new Whitelist.Struct for the white-list.
-                    // Expects the following format: 'class' ID -> ID '{' '\n'
+                    // Expects the following format: 'class' ID 'only_fqn'? '{' '\n'
                     if (line.startsWith("class ")) {
                         // Ensure the final token of the line is '{'.
                         if (line.endsWith("{") == false) {
@@ -153,17 +166,18 @@ public final class WhitelistLoader {
                                 "invalid struct definition: failed to parse class opening bracket [" + line + "]");
                         }
 
-                        // Parse the Painless type name and Java class name.
-                        String[] tokens = line.substring(5, line.length() - 1).replaceAll("\\s+", "").split("->");
+                        // Parse the Java class name.
+                        String[] tokens = line.substring(5, line.length() - 1).trim().split("\\s+");
 
                         // Ensure the correct number of tokens.
-                        if (tokens.length != 2) {
+                        if (tokens.length == 2 && "only_fqn".equals(tokens[1])) {
+                            onlyFQNJavaClassName = true;
+                        } else if (tokens.length != 1) {
                             throw new IllegalArgumentException("invalid struct definition: failed to parse class name [" + line + "]");
                         }
 
                         whitelistStructOrigin = "[" + filepath + "]:[" + number + "]";
-                        painlessTypeName = tokens[0];
-                        javaClassName = tokens[1];
+                        javaClassName = tokens[0];
 
                         // Reset all the constructors, methods, and fields to support a new struct.
                         whitelistConstructors = new ArrayList<>();
@@ -174,17 +188,17 @@ public final class WhitelistLoader {
                     // constructors, methods, augmented methods, and fields, and adding it to the list of white-listed structs.
                     // Expects the following format: '}' '\n'
                     } else if (line.equals("}")) {
-                        if (painlessTypeName == null) {
+                        if (javaClassName == null) {
                             throw new IllegalArgumentException("invalid struct definition: extraneous closing bracket");
                         }
 
-                        whitelistStructs.add(new Whitelist.Struct(whitelistStructOrigin, painlessTypeName, javaClassName,
+                        whitelistStructs.add(new Whitelist.Struct(whitelistStructOrigin, javaClassName, onlyFQNJavaClassName,
                             whitelistConstructors, whitelistMethods, whitelistFields));
 
                         // Set all the variables to null to ensure a new struct definition is found before other parsable values.
                         whitelistStructOrigin = null;
-                        painlessTypeName = null;
                         javaClassName = null;
+                        onlyFQNJavaClassName = false;
                         whitelistConstructors = null;
                         whitelistMethods = null;
                         whitelistFields = null;
@@ -195,7 +209,7 @@ public final class WhitelistLoader {
                         String origin = "[" + filepath + "]:[" + number + "]";
 
                         // Ensure we have a defined struct before adding any constructors, methods, augmented methods, or fields.
-                        if (painlessTypeName == null) {
+                        if (javaClassName == null) {
                             throw new IllegalArgumentException("invalid object definition: expected a class name [" + line + "]");
                         }
 
@@ -229,7 +243,7 @@ public final class WhitelistLoader {
 
                             // Parse the tokens prior to the method parameters.
                             int parameterIndex = line.indexOf('(');
-                            String[] tokens = line.substring(0, parameterIndex).split("\\s+");
+                            String[] tokens = line.trim().substring(0, parameterIndex).split("\\s+");
 
                             String javaMethodName;
                             String javaAugmentedClassName;
@@ -275,7 +289,7 @@ public final class WhitelistLoader {
                 }
 
                 // Ensure all structs end with a '}' token before the end of the file.
-                if (painlessTypeName != null) {
+                if (javaClassName != null) {
                     throw new IllegalArgumentException("invalid struct definition: expected closing bracket");
                 }
             } catch (Exception exception) {
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
index a5a414008d7..a793ef847f9 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
@@ -24,14 +24,14 @@
 
 #### Interfaces
 
-class Appendable -> java.lang.Appendable {
+class java.lang.Appendable {
   # append(char/CharSequence): skipped. left to subclasses (e.g. StringBuilder).
   Appendable append(CharSequence,int,int)
 }
 
 # AutoCloseable: i/o
 
-class CharSequence -> java.lang.CharSequence {
+class java.lang.CharSequence {
   char charAt(int)
   IntStream chars()
   IntStream codePoints()
@@ -44,11 +44,11 @@ class CharSequence -> java.lang.CharSequence {
 
 # Cloneable: add clone() to subclasses directly.
 
-class Comparable -> java.lang.Comparable {
+class java.lang.Comparable {
   int compareTo(def)
 }
 
-class Iterable -> java.lang.Iterable {
+class java.lang.Iterable {
   void forEach(Consumer)
   Iterator iterator()
   Spliterator spliterator()
@@ -72,7 +72,7 @@ class Iterable -> java.lang.Iterable {
 
 #### Classes
 
-class Boolean -> java.lang.Boolean {
+class java.lang.Boolean {
   Boolean TRUE
   Boolean FALSE
   boolean booleanValue()
@@ -87,7 +87,7 @@ class Boolean -> java.lang.Boolean {
   Boolean valueOf(boolean)
 }
 
-class Byte -> java.lang.Byte {
+class java.lang.Byte {
   int BYTES
   byte MAX_VALUE
   byte MIN_VALUE
@@ -105,7 +105,7 @@ class Byte -> java.lang.Byte {
   Byte valueOf(String,int)
 }
 
-class Character -> java.lang.Character {
+class java.lang.Character {
   int BYTES
   byte COMBINING_SPACING_MARK
   byte CONNECTOR_PUNCTUATION
@@ -226,10 +226,10 @@ class Character -> java.lang.Character {
   Character valueOf(char)
 }
 
-class Character.Subset -> java.lang.Character$Subset {
+class java.lang.Character$Subset {
 }
 
-class Character.UnicodeBlock -> java.lang.Character$UnicodeBlock {
+class java.lang.Character$UnicodeBlock {
   Character.UnicodeBlock AEGEAN_NUMBERS
   Character.UnicodeBlock ALCHEMICAL_SYMBOLS
   Character.UnicodeBlock ALPHABETIC_PRESENTATION_FORMS
@@ -459,7 +459,7 @@ class Character.UnicodeBlock -> java.lang.Character$UnicodeBlock {
 # ClassValue: ...
 # Compiler: ...
 
-class Double -> java.lang.Double {
+class java.lang.Double {
   int BYTES
   int MAX_EXPONENT
   double MAX_VALUE
@@ -490,13 +490,13 @@ class Double -> java.lang.Double {
   Double valueOf(double)
 }
 
-class Enum -> java.lang.Enum {
+class java.lang.Enum {
   int compareTo(Enum)
   String name()
   int ordinal()
 }
 
-class Float -> java.lang.Float {
+class java.lang.Float {
   int BYTES
   int MAX_EXPONENT
   float MAX_VALUE
@@ -529,7 +529,7 @@ class Float -> java.lang.Float {
 
 # InheritableThreadLocal: threads
 
-class Integer -> java.lang.Integer {
+class java.lang.Integer {
   int BYTES
   int MAX_VALUE
   int MIN_VALUE
@@ -569,7 +569,7 @@ class Integer -> java.lang.Integer {
   Integer valueOf(String,int)
 }
 
-class Long -> java.lang.Long {
+class java.lang.Long {
   int BYTES
   long MAX_VALUE
   long MIN_VALUE
@@ -609,7 +609,7 @@ class Long -> java.lang.Long {
   Long valueOf(String,int)
 }
 
-class Math -> java.lang.Math {
+class java.lang.Math {
   double E
   double PI
   double abs(double)
@@ -651,7 +651,7 @@ class Math -> java.lang.Math {
   double ulp(double)
 }
 
-class Number -> java.lang.Number {
+class java.lang.Number {
   byte byteValue()
   short shortValue()
   int intValue()
@@ -660,7 +660,7 @@ class Number -> java.lang.Number {
   double doubleValue()
 }
 
-class Object -> java.lang.Object {
+class java.lang.Object {
   boolean equals(Object)
   int hashCode()
   String toString()
@@ -674,7 +674,7 @@ class Object -> java.lang.Object {
 # RuntimePermission: skipped
 # SecurityManger: skipped
 
-class Short -> java.lang.Short {
+class java.lang.Short {
   int BYTES
   short MAX_VALUE
   short MIN_VALUE
@@ -693,7 +693,7 @@ class Short -> java.lang.Short {
   Short valueOf(String,int)
 }
 
-class StackTraceElement -> java.lang.StackTraceElement {
+class java.lang.StackTraceElement {
   (String,String,String,int)
   String getClassName()
   String getFileName()
@@ -702,7 +702,7 @@ class StackTraceElement -> java.lang.StackTraceElement {
   boolean isNativeMethod()
 }
 
-class StrictMath -> java.lang.StrictMath {
+class java.lang.StrictMath {
   double E
   double PI
   double abs(double)
@@ -744,7 +744,7 @@ class StrictMath -> java.lang.StrictMath {
   double ulp(double)
 }
 
-class String -> java.lang.String {
+class java.lang.String {
   ()
   int codePointAt(int)
   int codePointBefore(int)
@@ -786,7 +786,7 @@ class String -> java.lang.String {
   String valueOf(def)
 }
 
-class StringBuffer -> java.lang.StringBuffer {
+class java.lang.StringBuffer {
   ()
   (CharSequence)
   StringBuffer append(def)
@@ -813,7 +813,7 @@ class StringBuffer -> java.lang.StringBuffer {
   String substring(int,int)
 }
 
-class StringBuilder -> java.lang.StringBuilder {
+class java.lang.StringBuilder {
   ()
   (CharSequence)
   StringBuilder append(def)
@@ -840,7 +840,7 @@ class StringBuilder -> java.lang.StringBuilder {
   String substring(int,int)
 }
 
-class System -> java.lang.System {
+class java.lang.System {
   void arraycopy(Object,int,Object,int,int)
   long currentTimeMillis()
   long nanoTime()
@@ -851,12 +851,12 @@ class System -> java.lang.System {
 # ThreadLocal: skipped
 # Throwable: skipped (reserved for painless, users can only catch Exceptions)
 
-class Void -> java.lang.Void {
+class java.lang.Void {
 }
 
 #### Enums
 
-class Character.UnicodeScript -> java.lang.Character$UnicodeScript {
+class java.lang.Character$UnicodeScript {
   Character.UnicodeScript ARABIC
   Character.UnicodeScript ARMENIAN
   Character.UnicodeScript AVESTAN
@@ -968,41 +968,41 @@ class Character.UnicodeScript -> java.lang.Character$UnicodeScript {
 
 #### Exceptions
 
-class ArithmeticException -> java.lang.ArithmeticException {
+class java.lang.ArithmeticException {
   ()
   (String)
 }
 
-class ArrayIndexOutOfBoundsException -> java.lang.ArrayIndexOutOfBoundsException {
+class java.lang.ArrayIndexOutOfBoundsException {
   ()
   (String)
 }
 
-class ArrayStoreException -> java.lang.ArrayStoreException {
+class java.lang.ArrayStoreException {
   ()
   (String)
 }
 
-class ClassCastException -> java.lang.ClassCastException {
+class java.lang.ClassCastException {
   ()
   (String)
 }
 
-class ClassNotFoundException -> java.lang.ClassNotFoundException {
+class java.lang.ClassNotFoundException {
   ()
   (String)
 }
 
-class CloneNotSupportedException -> java.lang.CloneNotSupportedException {
+class java.lang.CloneNotSupportedException {
   ()
   (String)
 }
 
-class EnumConstantNotPresentException -> java.lang.EnumConstantNotPresentException {
+class java.lang.EnumConstantNotPresentException {
   String constantName()
 }
 
-class Exception -> java.lang.Exception {
+class java.lang.Exception {
   ()
   (String)
   String getLocalizedMessage()
@@ -1010,96 +1010,96 @@ class Exception -> java.lang.Exception {
   StackTraceElement[] getStackTrace()
 }
 
-class IllegalAccessException -> java.lang.IllegalAccessException {
+class java.lang.IllegalAccessException {
   ()
   (String)
 }
 
-class IllegalArgumentException -> java.lang.IllegalArgumentException {
+class java.lang.IllegalArgumentException {
   ()
   (String)
 }
 
-class IllegalMonitorStateException -> java.lang.IllegalMonitorStateException {
+class java.lang.IllegalMonitorStateException {
   ()
   (String)
 }
 
-class IllegalStateException -> java.lang.IllegalStateException {
+class java.lang.IllegalStateException {
   ()
   (String)
 }
 
-class IllegalThreadStateException -> java.lang.IllegalThreadStateException {
+class java.lang.IllegalThreadStateException {
   ()
   (String)
 }
 
-class IndexOutOfBoundsException -> java.lang.IndexOutOfBoundsException {
+class java.lang.IndexOutOfBoundsException {
   ()
   (String)
 }
 
-class InstantiationException -> java.lang.InstantiationException {
+class java.lang.InstantiationException {
   ()
   (String)
 }
 
-class InterruptedException -> java.lang.InterruptedException {
+class java.lang.InterruptedException {
   ()
   (String)
 }
 
-class NegativeArraySizeException -> java.lang.NegativeArraySizeException {
+class java.lang.NegativeArraySizeException {
   ()
   (String)
 }
 
-class NoSuchFieldException -> java.lang.NoSuchFieldException {
+class java.lang.NoSuchFieldException {
   ()
   (String)
 }
 
-class NoSuchMethodException -> java.lang.NoSuchMethodException {
+class java.lang.NoSuchMethodException {
   ()
   (String)
 }
 
-class NullPointerException -> java.lang.NullPointerException {
+class java.lang.NullPointerException {
   ()
   (String)
 }
 
-class NumberFormatException -> java.lang.NumberFormatException {
+class java.lang.NumberFormatException {
   ()
   (String)
 }
 
-class ReflectiveOperationException -> java.lang.ReflectiveOperationException {
+class java.lang.ReflectiveOperationException {
   ()
   (String)
 }
 
-class RuntimeException -> java.lang.RuntimeException {
+class java.lang.RuntimeException {
   ()
   (String)
 }
 
-class SecurityException -> java.lang.SecurityException {
+class java.lang.SecurityException {
   ()
   (String)
 }
 
-class StringIndexOutOfBoundsException -> java.lang.StringIndexOutOfBoundsException {
+class java.lang.StringIndexOutOfBoundsException {
   ()
   (String)
 }
 
-class TypeNotPresentException -> java.lang.TypeNotPresentException {
+class java.lang.TypeNotPresentException {
   String typeName()
 }
 
-class UnsupportedOperationException -> java.lang.UnsupportedOperationException {
+class java.lang.UnsupportedOperationException {
   ()
   (String)
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt
index e7457628203..e66c6d8c91f 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt
@@ -24,7 +24,7 @@
 
 #### Classes
 
-class BigDecimal -> java.math.BigDecimal {
+class java.math.BigDecimal {
   BigDecimal ONE
   BigDecimal TEN
   BigDecimal ZERO
@@ -77,7 +77,7 @@ class BigDecimal -> java.math.BigDecimal {
   BigDecimal valueOf(double)
 }
 
-class BigInteger -> java.math.BigInteger {
+class java.math.BigInteger {
   BigInteger ONE
   BigInteger TEN
   BigInteger ZERO
@@ -123,7 +123,7 @@ class BigInteger -> java.math.BigInteger {
   BigInteger xor(BigInteger)
 }
 
-class MathContext -> java.math.MathContext {
+class java.math.MathContext {
   MathContext DECIMAL128
   MathContext DECIMAL32
   MathContext DECIMAL64
@@ -136,7 +136,7 @@ class MathContext -> java.math.MathContext {
 
 #### Enums
 
-class RoundingMode -> java.math.RoundingMode {
+class java.math.RoundingMode {
   RoundingMode CEILING
   RoundingMode DOWN
   RoundingMode FLOOR
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt
index fa9170cb5d2..2b159123953 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt
@@ -24,7 +24,7 @@
 
 #### Interfaces
 
-class AttributedCharacterIterator -> java.text.AttributedCharacterIterator {
+class java.text.AttributedCharacterIterator {
   Set getAllAttributeKeys()
   def getAttribute(AttributedCharacterIterator.Attribute)
   Map getAttributes()
@@ -34,7 +34,7 @@ class AttributedCharacterIterator -> java.text.AttributedCharacterIterator {
   int getRunStart(Set)
 }
 
-class CharacterIterator -> java.text.CharacterIterator {
+class java.text.CharacterIterator {
   char DONE
   def clone()
   char current()
@@ -50,18 +50,18 @@ class CharacterIterator -> java.text.CharacterIterator {
 
 #### Classes
 
-class Annotation -> java.text.Annotation {
+class java.text.Annotation {
   (Object)
   def getValue()
 }
 
-class AttributedCharacterIterator.Attribute -> java.text.AttributedCharacterIterator$Attribute {
+class java.text.AttributedCharacterIterator$Attribute {
   AttributedCharacterIterator.Attribute INPUT_METHOD_SEGMENT
   AttributedCharacterIterator.Attribute LANGUAGE
   AttributedCharacterIterator.Attribute READING
 }
 
-class AttributedString -> java.text.AttributedString {
+class java.text.AttributedString {
   (String)
   (String,Map)
   void addAttribute(AttributedCharacterIterator.Attribute,Object)
@@ -72,7 +72,7 @@ class AttributedString -> java.text.AttributedString {
   AttributedCharacterIterator getIterator(AttributedCharacterIterator.Attribute[],int,int)
 }
 
-class Bidi -> java.text.Bidi {
+class java.text.Bidi {
   int DIRECTION_DEFAULT_LEFT_TO_RIGHT
   int DIRECTION_DEFAULT_RIGHT_TO_LEFT
   int DIRECTION_LEFT_TO_RIGHT
@@ -96,7 +96,7 @@ class Bidi -> java.text.Bidi {
   boolean requiresBidi(char[],int,int)
 }
 
-class BreakIterator -> java.text.BreakIterator {
+class java.text.BreakIterator {
   int DONE
   def clone()
   int current()
@@ -121,7 +121,7 @@ class BreakIterator -> java.text.BreakIterator {
   void setText(String)
 }
 
-class ChoiceFormat -> java.text.ChoiceFormat {
+class java.text.ChoiceFormat {
   (double[],String[])
   (String)
   void applyPattern(String)
@@ -134,7 +134,7 @@ class ChoiceFormat -> java.text.ChoiceFormat {
   String toPattern()
 }
 
-class CollationElementIterator -> java.text.CollationElementIterator {
+class java.text.CollationElementIterator {
   int NULLORDER
   int getMaxExpansion(int)
   int getOffset()
@@ -148,13 +148,13 @@ class CollationElementIterator -> java.text.CollationElementIterator {
   short tertiaryOrder(int)
 }
 
-class CollationKey -> java.text.CollationKey {
+class java.text.CollationKey {
   int compareTo(CollationKey)
   String getSourceString()
   byte[] toByteArray()
 }
 
-class Collator -> java.text.Collator {
+class java.text.Collator {
   int CANONICAL_DECOMPOSITION
   int FULL_DECOMPOSITION
   int IDENTICAL
@@ -174,7 +174,7 @@ class Collator -> java.text.Collator {
   void setStrength(int)
 }
 
-class DateFormat -> java.text.DateFormat {
+class java.text.DateFormat {
   int AM_PM_FIELD
   int DATE_FIELD
   int DAY_OF_WEEK_FIELD
@@ -221,7 +221,7 @@ class DateFormat -> java.text.DateFormat {
   void setTimeZone(TimeZone)
 }
 
-class DateFormat.Field -> java.text.DateFormat$Field {
+class java.text.DateFormat$Field {
   DateFormat.Field AM_PM
   DateFormat.Field DAY_OF_MONTH
   DateFormat.Field DAY_OF_WEEK
@@ -244,7 +244,7 @@ class DateFormat.Field -> java.text.DateFormat$Field {
   DateFormat.Field ofCalendarField(int)
 }
 
-class DateFormatSymbols -> java.text.DateFormatSymbols {
+class java.text.DateFormatSymbols {
   ()
   (Locale)
   def clone()
@@ -270,7 +270,7 @@ class DateFormatSymbols -> java.text.DateFormatSymbols {
   void setZoneStrings(String[][])
 }
 
-class DecimalFormat -> java.text.DecimalFormat {
+class java.text.DecimalFormat {
   ()
   (String)
   (String,DecimalFormatSymbols)
@@ -298,7 +298,7 @@ class DecimalFormat -> java.text.DecimalFormat {
   String toPattern()
 }
 
-class DecimalFormatSymbols -> java.text.DecimalFormatSymbols {
+class java.text.DecimalFormatSymbols {
   ()
   (Locale)
   def clone()
@@ -337,7 +337,7 @@ class DecimalFormatSymbols -> java.text.DecimalFormatSymbols {
   void setZeroDigit(char)
 }
 
-class FieldPosition -> java.text.FieldPosition {
+class java.text.FieldPosition {
   (int)
   (Format.Field,int)
   int getBeginIndex()
@@ -348,7 +348,7 @@ class FieldPosition -> java.text.FieldPosition {
   void setEndIndex(int)
 }
 
-class Format -> java.text.Format {
+class java.text.Format {
   def clone()
   String format(Object)
   StringBuffer format(Object,StringBuffer,FieldPosition)
@@ -357,10 +357,10 @@ class Format -> java.text.Format {
   Object parseObject(String,ParsePosition)
 }
 
-class Format.Field -> java.text.Format$Field {
+class java.text.Format$Field {
 }
 
-class MessageFormat -> java.text.MessageFormat {
+class java.text.MessageFormat {
   void applyPattern(String)
   String format(String,Object[])
   Format[] getFormats()
@@ -376,16 +376,16 @@ class MessageFormat -> java.text.MessageFormat {
   String toPattern()
 }
 
-class MessageFormat.Field -> java.text.MessageFormat$Field {
+class java.text.MessageFormat$Field {
   MessageFormat.Field ARGUMENT
 }
 
-class Normalizer -> java.text.Normalizer {
+class java.text.Normalizer {
   boolean isNormalized(CharSequence,Normalizer.Form)
   String normalize(CharSequence,Normalizer.Form)
 }
 
-class NumberFormat -> java.text.NumberFormat {
+class java.text.NumberFormat {
   int FRACTION_FIELD
   int INTEGER_FIELD
   Locale[] getAvailableLocales()
@@ -419,7 +419,7 @@ class NumberFormat -> java.text.NumberFormat {
   void setRoundingMode(RoundingMode)
 }
 
-class NumberFormat.Field -> java.text.NumberFormat$Field {
+class java.text.NumberFormat$Field {
   NumberFormat.Field CURRENCY
   NumberFormat.Field DECIMAL_SEPARATOR
   NumberFormat.Field EXPONENT
@@ -433,7 +433,7 @@ class NumberFormat.Field -> java.text.NumberFormat$Field {
   NumberFormat.Field SIGN
 }
 
-class ParsePosition -> java.text.ParsePosition {
+class java.text.ParsePosition {
   (int)
   int getErrorIndex()
   int getIndex()
@@ -441,13 +441,13 @@ class ParsePosition -> java.text.ParsePosition {
   void setIndex(int)
 }
 
-class RuleBasedCollator -> java.text.RuleBasedCollator {
+class java.text.RuleBasedCollator {
   (String)
   CollationElementIterator getCollationElementIterator(String)
   String getRules()
 }
 
-class SimpleDateFormat -> java.text.SimpleDateFormat {
+class java.text.SimpleDateFormat {
   ()
   (String)
   (String,Locale)
@@ -461,7 +461,7 @@ class SimpleDateFormat -> java.text.SimpleDateFormat {
   String toPattern()
 }
 
-class StringCharacterIterator -> java.text.StringCharacterIterator {
+class java.text.StringCharacterIterator {
   (String)
   (String,int)
   (String,int,int,int)
@@ -470,7 +470,7 @@ class StringCharacterIterator -> java.text.StringCharacterIterator {
 
 #### Enums
 
-class Normalizer.Form -> java.text.Normalizer$Form {
+class java.text.Normalizer$Form {
   Normalizer.Form NFC
   Normalizer.Form NFD
   Normalizer.Form NFKC
@@ -481,7 +481,7 @@ class Normalizer.Form -> java.text.Normalizer$Form {
 
 #### Exceptions
 
-class ParseException -> java.text.ParseException {
+class java.text.ParseException {
   (String,int)
   int getErrorOffset()
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
index 2d932a3ed1a..25740ff9d3b 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
@@ -24,7 +24,7 @@
 
 #### Interfaces
 
-class ChronoLocalDate -> java.time.chrono.ChronoLocalDate {
+class java.time.chrono.ChronoLocalDate {
   ChronoLocalDateTime atTime(LocalTime)
   int compareTo(ChronoLocalDate)
   boolean equals(Object)
@@ -51,7 +51,7 @@ class ChronoLocalDate -> java.time.chrono.ChronoLocalDate {
   ChronoLocalDate with(TemporalField,long)
 }
 
-class ChronoLocalDateTime -> java.time.chrono.ChronoLocalDateTime {
+class java.time.chrono.ChronoLocalDateTime {
   ChronoZonedDateTime atZone(ZoneId)
   int compareTo(ChronoLocalDateTime)
   boolean equals(Object)
@@ -76,7 +76,7 @@ class ChronoLocalDateTime -> java.time.chrono.ChronoLocalDateTime {
   ChronoLocalDateTime with(TemporalField,long)
 }
 
-class Chronology -> java.time.chrono.Chronology {
+class java.time.chrono.Chronology {
   int compareTo(Chronology)
   ChronoLocalDate date(TemporalAccessor)
   ChronoLocalDate date(Era,int,int,int)
@@ -106,7 +106,7 @@ class Chronology -> java.time.chrono.Chronology {
   ChronoZonedDateTime zonedDateTime(Instant,ZoneId)
 }
 
-class ChronoPeriod -> java.time.chrono.ChronoPeriod {
+class java.time.chrono.ChronoPeriod {
   ChronoPeriod between(ChronoLocalDate,ChronoLocalDate)
   boolean equals(Object)
   Chronology getChronology()
@@ -122,7 +122,7 @@ class ChronoPeriod -> java.time.chrono.ChronoPeriod {
   String toString()
 }
 
-class ChronoZonedDateTime -> java.time.chrono.ChronoZonedDateTime {
+class java.time.chrono.ChronoZonedDateTime {
   int compareTo(ChronoZonedDateTime)
   boolean equals(Object)
   String format(DateTimeFormatter)
@@ -153,17 +153,17 @@ class ChronoZonedDateTime -> java.time.chrono.ChronoZonedDateTime {
   ChronoZonedDateTime withZoneSameInstant(ZoneId)
 }
 
-class Era -> java.time.chrono.Era {
+class java.time.chrono.Era {
   String getDisplayName(TextStyle,Locale)
   int getValue()
 }
 
 #### Classes
 
-class AbstractChronology -> java.time.chrono.AbstractChronology {
+class java.time.chrono.AbstractChronology {
 }
 
-class HijrahChronology -> java.time.chrono.HijrahChronology {
+class java.time.chrono.HijrahChronology {
   HijrahChronology INSTANCE
   HijrahDate date(TemporalAccessor)
   HijrahDate date(int,int,int)
@@ -175,7 +175,7 @@ class HijrahChronology -> java.time.chrono.HijrahChronology {
   HijrahDate resolveDate(Map,ResolverStyle)
 }
 
-class HijrahDate -> java.time.chrono.HijrahDate {
+class java.time.chrono.HijrahDate {
   HijrahDate from(TemporalAccessor)
   HijrahChronology getChronology()
   HijrahEra getEra()
@@ -189,7 +189,7 @@ class HijrahDate -> java.time.chrono.HijrahDate {
   HijrahDate withVariant(HijrahChronology)
 }
 
-class IsoChronology -> java.time.chrono.IsoChronology {
+class java.time.chrono.IsoChronology {
   IsoChronology INSTANCE
   LocalDate date(TemporalAccessor)
   LocalDate date(int,int,int)
@@ -205,7 +205,7 @@ class IsoChronology -> java.time.chrono.IsoChronology {
   ZonedDateTime zonedDateTime(Instant,ZoneId)
 }
 
-class JapaneseChronology -> java.time.chrono.JapaneseChronology {
+class java.time.chrono.JapaneseChronology {
   JapaneseChronology INSTANCE
   JapaneseDate date(TemporalAccessor)
   JapaneseDate date(int,int,int)
@@ -217,7 +217,7 @@ class JapaneseChronology -> java.time.chrono.JapaneseChronology {
   JapaneseDate resolveDate(Map,ResolverStyle)
 }
 
-class JapaneseDate -> java.time.chrono.JapaneseDate {
+class java.time.chrono.JapaneseDate {
   JapaneseDate of(int,int,int)
   JapaneseDate from(TemporalAccessor)
   JapaneseChronology getChronology()
@@ -230,7 +230,7 @@ class JapaneseDate -> java.time.chrono.JapaneseDate {
   JapaneseDate minus(long,TemporalUnit)
 }
 
-class JapaneseEra -> java.time.chrono.JapaneseEra {
+class java.time.chrono.JapaneseEra {
   JapaneseEra HEISEI
   JapaneseEra MEIJI
   JapaneseEra SHOWA
@@ -241,7 +241,7 @@ class JapaneseEra -> java.time.chrono.JapaneseEra {
   JapaneseEra[] values()
 }
 
-class MinguoChronology -> java.time.chrono.MinguoChronology {
+class java.time.chrono.MinguoChronology {
   MinguoChronology INSTANCE
   MinguoDate date(TemporalAccessor)
   MinguoDate date(int,int,int)
@@ -253,7 +253,7 @@ class MinguoChronology -> java.time.chrono.MinguoChronology {
   MinguoDate resolveDate(Map,ResolverStyle)
 }
 
-class MinguoDate -> java.time.chrono.MinguoDate {
+class java.time.chrono.MinguoDate {
   MinguoDate of(int,int,int)
   MinguoDate from(TemporalAccessor)
   MinguoChronology getChronology()
@@ -266,7 +266,7 @@ class MinguoDate -> java.time.chrono.MinguoDate {
   MinguoDate minus(long,TemporalUnit)
 }
 
-class ThaiBuddhistChronology -> java.time.chrono.ThaiBuddhistChronology {
+class java.time.chrono.ThaiBuddhistChronology {
   ThaiBuddhistChronology INSTANCE
   ThaiBuddhistDate date(TemporalAccessor)
   ThaiBuddhistDate date(int,int,int)
@@ -278,7 +278,7 @@ class ThaiBuddhistChronology -> java.time.chrono.ThaiBuddhistChronology {
   ThaiBuddhistDate resolveDate(Map,ResolverStyle)
 }
 
-class ThaiBuddhistDate -> java.time.chrono.ThaiBuddhistDate {
+class java.time.chrono.ThaiBuddhistDate {
   ThaiBuddhistDate of(int,int,int)
   ThaiBuddhistDate from(TemporalAccessor)
   ThaiBuddhistChronology getChronology()
@@ -293,7 +293,7 @@ class ThaiBuddhistDate -> java.time.chrono.ThaiBuddhistDate {
 
 #### Enums
 
-class HijrahEra -> java.time.chrono.HijrahEra {
+class java.time.chrono.HijrahEra {
   HijrahEra AH
   int getValue()
   HijrahEra of(int)
@@ -301,7 +301,7 @@ class HijrahEra -> java.time.chrono.HijrahEra {
   HijrahEra[] values()
 }
 
-class IsoEra -> java.time.chrono.IsoEra {
+class java.time.chrono.IsoEra {
   IsoEra BCE
   IsoEra CE
   int getValue()
@@ -310,7 +310,7 @@ class IsoEra -> java.time.chrono.IsoEra {
   IsoEra[] values()
 }
 
-class MinguoEra -> java.time.chrono.MinguoEra {
+class java.time.chrono.MinguoEra {
   MinguoEra BEFORE_ROC
   MinguoEra ROC
   int getValue()
@@ -319,7 +319,7 @@ class MinguoEra -> java.time.chrono.MinguoEra {
   MinguoEra[] values()
 }
 
-class ThaiBuddhistEra -> java.time.chrono.ThaiBuddhistEra {
+class java.time.chrono.ThaiBuddhistEra {
   ThaiBuddhistEra BE
   ThaiBuddhistEra BEFORE_BE
   int getValue()
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt
index d5b5c9cc355..59f484454a7 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt
@@ -24,7 +24,7 @@
 
 #### Classes
 
-class DateTimeFormatter -> java.time.format.DateTimeFormatter {
+class java.time.format.DateTimeFormatter {
   DateTimeFormatter BASIC_ISO_DATE
   DateTimeFormatter ISO_DATE
   DateTimeFormatter ISO_DATE_TIME
@@ -70,7 +70,7 @@ class DateTimeFormatter -> java.time.format.DateTimeFormatter {
   DateTimeFormatter withZone(ZoneId)
 }
 
-class DateTimeFormatterBuilder -> java.time.format.DateTimeFormatterBuilder {
+class java.time.format.DateTimeFormatterBuilder {
   ()
   DateTimeFormatterBuilder append(DateTimeFormatter)
   DateTimeFormatterBuilder appendChronologyId()
@@ -110,7 +110,7 @@ class DateTimeFormatterBuilder -> java.time.format.DateTimeFormatterBuilder {
   DateTimeFormatter toFormatter(Locale)
 }
 
-class DecimalStyle -> java.time.format.DecimalStyle {
+class java.time.format.DecimalStyle {
   DecimalStyle STANDARD
   Set getAvailableLocales()
   char getDecimalSeparator()
@@ -127,7 +127,7 @@ class DecimalStyle -> java.time.format.DecimalStyle {
 
 #### Enums
 
-class FormatStyle -> java.time.format.FormatStyle {
+class java.time.format.FormatStyle {
   FormatStyle FULL
   FormatStyle LONG
   FormatStyle MEDIUM
@@ -136,7 +136,7 @@ class FormatStyle -> java.time.format.FormatStyle {
   FormatStyle[] values()
 }
 
-class ResolverStyle -> java.time.format.ResolverStyle {
+class java.time.format.ResolverStyle {
   ResolverStyle LENIENT
   ResolverStyle SMART
   ResolverStyle STRICT
@@ -144,7 +144,7 @@ class ResolverStyle -> java.time.format.ResolverStyle {
   ResolverStyle[] values()
 }
 
-class SignStyle -> java.time.format.SignStyle {
+class java.time.format.SignStyle {
   SignStyle ALWAYS
   SignStyle EXCEEDS_PAD
   SignStyle NEVER
@@ -154,7 +154,7 @@ class SignStyle -> java.time.format.SignStyle {
   SignStyle[] values()
 }
 
-class TextStyle -> java.time.format.TextStyle {
+class java.time.format.TextStyle {
   TextStyle FULL
   TextStyle FULL_STANDALONE
   TextStyle NARROW
@@ -170,7 +170,7 @@ class TextStyle -> java.time.format.TextStyle {
 
 #### Exceptions
 
-class DateTimeParseException -> java.time.format.DateTimeParseException {
+class java.time.format.DateTimeParseException {
   (String,CharSequence,int)
   int getErrorIndex()
   String getParsedString()
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt
index e3c09bc6255..3ef2d33accc 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt
@@ -24,7 +24,7 @@
 
 #### Interfaces
 
-class Temporal -> java.time.temporal.Temporal {
+class java.time.temporal.Temporal {
   Temporal minus(long,TemporalUnit)
   Temporal minus(TemporalAmount)
   Temporal plus(long,TemporalUnit)
@@ -34,7 +34,7 @@ class Temporal -> java.time.temporal.Temporal {
   Temporal with(TemporalField,long)
 }
 
-class TemporalAccessor -> java.time.temporal.TemporalAccessor {
+class java.time.temporal.TemporalAccessor {
   int get(TemporalField)
   long getLong(TemporalField)
   boolean isSupported(TemporalField)
@@ -42,18 +42,18 @@ class TemporalAccessor -> java.time.temporal.TemporalAccessor {
   ValueRange range(TemporalField)
 }
 
-class TemporalAdjuster -> java.time.temporal.TemporalAdjuster {
+class java.time.temporal.TemporalAdjuster {
   Temporal adjustInto(Temporal)
 }
 
-class TemporalAmount -> java.time.temporal.TemporalAmount {
+class java.time.temporal.TemporalAmount {
   Temporal addTo(Temporal)
   long get(TemporalUnit)
   List getUnits()
   Temporal subtractFrom(Temporal)
 }
 
-class TemporalField -> java.time.temporal.TemporalField {
+class java.time.temporal.TemporalField {
   Temporal adjustInto(Temporal,long)
   TemporalUnit getBaseUnit()
   String getDisplayName(Locale)
@@ -68,11 +68,11 @@ class TemporalField -> java.time.temporal.TemporalField {
   String toString()
 }
 
-class TemporalQuery -> java.time.temporal.TemporalQuery {
+class java.time.temporal.TemporalQuery {
   def queryFrom(TemporalAccessor)
 }
 
-class TemporalUnit -> java.time.temporal.TemporalUnit {
+class java.time.temporal.TemporalUnit {
   Temporal addTo(Temporal,long)
   long between(Temporal,Temporal)
   Duration getDuration()
@@ -85,7 +85,7 @@ class TemporalUnit -> java.time.temporal.TemporalUnit {
 
 #### Classes
 
-class IsoFields -> java.time.temporal.IsoFields {
+class java.time.temporal.IsoFields {
   TemporalField DAY_OF_QUARTER
   TemporalField QUARTER_OF_YEAR
   TemporalUnit QUARTER_YEARS
@@ -94,13 +94,13 @@ class IsoFields -> java.time.temporal.IsoFields {
   TemporalField WEEK_OF_WEEK_BASED_YEAR
 }
 
-class JulianFields -> java.time.temporal.JulianFields {
+class java.time.temporal.JulianFields {
   TemporalField JULIAN_DAY
   TemporalField MODIFIED_JULIAN_DAY
   TemporalField RATA_DIE
 }
 
-class TemporalAdjusters -> java.time.temporal.TemporalAdjusters {
+class java.time.temporal.TemporalAdjusters {
   TemporalAdjuster dayOfWeekInMonth(int,DayOfWeek)
   TemporalAdjuster firstDayOfMonth()
   TemporalAdjuster firstDayOfNextMonth()
@@ -117,7 +117,7 @@ class TemporalAdjusters -> java.time.temporal.TemporalAdjusters {
   TemporalAdjuster previousOrSame(DayOfWeek)
 }
 
-class TemporalQueries -> java.time.temporal.TemporalQueries {
+class java.time.temporal.TemporalQueries {
   TemporalQuery chronology()
   TemporalQuery localDate()
   TemporalQuery localTime()
@@ -127,7 +127,7 @@ class TemporalQueries -> java.time.temporal.TemporalQueries {
   TemporalQuery zoneId()
 }
 
-class ValueRange -> java.time.temporal.ValueRange {
+class java.time.temporal.ValueRange {
   int checkValidIntValue(long,TemporalField)
   long checkValidValue(long,TemporalField)
   long getLargestMinimum()
@@ -143,7 +143,7 @@ class ValueRange -> java.time.temporal.ValueRange {
   ValueRange of(long,long,long,long)
 }
 
-class WeekFields -> java.time.temporal.WeekFields {
+class java.time.temporal.WeekFields {
   WeekFields ISO
   WeekFields SUNDAY_START
   TemporalUnit WEEK_BASED_YEARS
@@ -160,7 +160,7 @@ class WeekFields -> java.time.temporal.WeekFields {
 
 #### Enums
 
-class ChronoField -> java.time.temporal.ChronoField {
+class java.time.temporal.ChronoField {
   ChronoField ALIGNED_DAY_OF_WEEK_IN_MONTH
   ChronoField ALIGNED_DAY_OF_WEEK_IN_YEAR
   ChronoField ALIGNED_WEEK_OF_MONTH
@@ -197,7 +197,7 @@ class ChronoField -> java.time.temporal.ChronoField {
   ChronoField[] values()
 }
 
-class ChronoUnit -> java.time.temporal.ChronoUnit {
+class java.time.temporal.ChronoUnit {
   ChronoUnit CENTURIES
   ChronoUnit DAYS
   ChronoUnit DECADES
@@ -220,6 +220,6 @@ class ChronoUnit -> java.time.temporal.ChronoUnit {
 
 #### Exceptions
 
-class UnsupportedTemporalTypeException -> java.time.temporal.UnsupportedTemporalTypeException {
+class java.time.temporal.UnsupportedTemporalTypeException {
   (String)
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
index 1c012042b02..0cedc849a68 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
@@ -24,7 +24,7 @@
 
 #### Classes
 
-class Clock -> java.time.Clock {
+class java.time.Clock {
   Clock fixed(Instant,ZoneId)
   ZoneId getZone()
   Instant instant()
@@ -33,7 +33,7 @@ class Clock -> java.time.Clock {
   Clock tick(Clock,Duration)
 }
 
-class Duration -> java.time.Duration {
+class java.time.Duration {
   Duration ZERO
   Duration abs()
   Duration between(Temporal,Temporal)
@@ -80,7 +80,7 @@ class Duration -> java.time.Duration {
   Duration withNanos(int)
 }
 
-class Instant -> java.time.Instant {
+class java.time.Instant {
   Instant EPOCH
   Instant MAX
   Instant MIN
@@ -112,7 +112,7 @@ class Instant -> java.time.Instant {
   Instant with(TemporalField,long)
 }
 
-class LocalDate -> java.time.LocalDate {
+class java.time.LocalDate {
   LocalDate MAX
   LocalDate MIN
   LocalDateTime atStartOfDay()
@@ -155,7 +155,7 @@ class LocalDate -> java.time.LocalDate {
   LocalDate withYear(int)
 }
 
-class LocalDateTime -> java.time.LocalDateTime {
+class java.time.LocalDateTime {
   LocalDateTime MIN
   LocalDateTime MAX
   OffsetDateTime atOffset(ZoneOffset)
@@ -212,7 +212,7 @@ class LocalDateTime -> java.time.LocalDateTime {
   LocalDateTime withYear(int)
 }
 
-class LocalTime -> java.time.LocalTime {
+class java.time.LocalTime {
   LocalTime MAX
   LocalTime MIDNIGHT
   LocalTime MIN
@@ -258,7 +258,7 @@ class LocalTime -> java.time.LocalTime {
   LocalTime withSecond(int)
 }
 
-class MonthDay -> java.time.MonthDay {
+class java.time.MonthDay {
   LocalDate atYear(int)
   int compareTo(MonthDay)
   String format(DateTimeFormatter)
@@ -277,7 +277,7 @@ class MonthDay -> java.time.MonthDay {
   MonthDay withMonth(int)
 }
 
-class OffsetDateTime -> java.time.OffsetDateTime {
+class java.time.OffsetDateTime {
   OffsetDateTime MAX
   OffsetDateTime MIN
   ZonedDateTime atZoneSameInstant(ZoneId)
@@ -348,7 +348,7 @@ class OffsetDateTime -> java.time.OffsetDateTime {
   OffsetDateTime withOffsetSameInstant(ZoneOffset)
 }
 
-class OffsetTime -> java.time.OffsetTime {
+class java.time.OffsetTime {
   OffsetTime MAX
   OffsetTime MIN
   int compareTo(OffsetTime)
@@ -391,7 +391,7 @@ class OffsetTime -> java.time.OffsetTime {
   OffsetTime withSecond(int)
 }
 
-class Period -> java.time.Period {
+class java.time.Period {
   Period ZERO
   Period between(LocalDate,LocalDate)
   Period from(TemporalAmount)
@@ -422,7 +422,7 @@ class Period -> java.time.Period {
   Period withYears(int)
 }
 
-class Year -> java.time.Year {
+class java.time.Year {
   int MAX_VALUE
   int MIN_VALUE
   LocalDate atDay(int)
@@ -450,7 +450,7 @@ class Year -> java.time.Year {
   Year with(TemporalField,long)
 }
 
-class YearMonth -> java.time.YearMonth {
+class java.time.YearMonth {
   LocalDate atDay(int)
   LocalDate atEndOfMonth()
   int compareTo(YearMonth)
@@ -482,7 +482,7 @@ class YearMonth -> java.time.YearMonth {
   YearMonth withMonth(int)
 }
 
-class ZonedDateTime -> java.time.ZonedDateTime {
+class java.time.ZonedDateTime {
   int getDayOfMonth()
   DayOfWeek getDayOfWeek()
   int getDayOfYear()
@@ -544,7 +544,7 @@ class ZonedDateTime -> java.time.ZonedDateTime {
   ZonedDateTime withZoneSameInstant(ZoneId)
 }
 
-class ZoneId -> java.time.ZoneId {
+class java.time.ZoneId {
   Map SHORT_IDS
   Set getAvailableZoneIds()
   ZoneId of(String)
@@ -558,7 +558,7 @@ class ZoneId -> java.time.ZoneId {
   ZoneRules getRules()
 }
 
-class ZoneOffset -> java.time.ZoneOffset {
+class java.time.ZoneOffset {
   ZoneOffset MAX
   ZoneOffset MIN
   ZoneOffset UTC
@@ -573,7 +573,7 @@ class ZoneOffset -> java.time.ZoneOffset {
 
 #### Enums
 
-class DayOfWeek -> java.time.DayOfWeek {
+class java.time.DayOfWeek {
   DayOfWeek FRIDAY
   DayOfWeek MONDAY
   DayOfWeek SATURDAY
@@ -591,7 +591,7 @@ class DayOfWeek -> java.time.DayOfWeek {
   DayOfWeek[] values()
 }
 
-class Month -> java.time.Month {
+class java.time.Month {
   Month APRIL
   Month AUGUST
   Month DECEMBER
@@ -621,7 +621,7 @@ class Month -> java.time.Month {
 
 #### Exceptions
 
-class DateTimeException -> java.time.DateTimeException {
+class java.time.DateTimeException {
   (String)
 }
 
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt
index dfb6fc7a807..59d623cc990 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt
@@ -24,7 +24,7 @@
 
 #### Classes
 
-class ZoneOffsetTransition -> java.time.zone.ZoneOffsetTransition {
+class java.time.zone.ZoneOffsetTransition {
   int compareTo(ZoneOffsetTransition)
   LocalDateTime getDateTimeAfter()
   LocalDateTime getDateTimeBefore()
@@ -39,7 +39,7 @@ class ZoneOffsetTransition -> java.time.zone.ZoneOffsetTransition {
   long toEpochSecond()
 }
 
-class ZoneOffsetTransitionRule -> java.time.zone.ZoneOffsetTransitionRule {
+class java.time.zone.ZoneOffsetTransitionRule {
   ZoneOffsetTransition createTransition(int)
   int getDayOfMonthIndicator()
   DayOfWeek getDayOfWeek()
@@ -53,7 +53,7 @@ class ZoneOffsetTransitionRule -> java.time.zone.ZoneOffsetTransitionRule {
   ZoneOffsetTransitionRule of(Month,int,DayOfWeek,LocalTime,boolean,ZoneOffsetTransitionRule.TimeDefinition,ZoneOffset,ZoneOffset,ZoneOffset)
 }
 
-class ZoneRules -> java.time.zone.ZoneRules {
+class java.time.zone.ZoneRules {
   Duration getDaylightSavings(Instant)
   ZoneOffset getOffset(Instant)
   ZoneOffset getStandardOffset(Instant)
@@ -70,7 +70,7 @@ class ZoneRules -> java.time.zone.ZoneRules {
   ZoneOffsetTransition previousTransition(Instant)
 }
 
-class ZoneRulesProvider -> java.time.zone.ZoneRulesProvider {
+class java.time.zone.ZoneRulesProvider {
   Set getAvailableZoneIds()
   ZoneRules getRules(String,boolean)
   NavigableMap getVersions(String)
@@ -78,7 +78,7 @@ class ZoneRulesProvider -> java.time.zone.ZoneRulesProvider {
 
 #### Enums
 
-class ZoneOffsetTransitionRule.TimeDefinition -> java.time.zone.ZoneOffsetTransitionRule$TimeDefinition {
+class java.time.zone.ZoneOffsetTransitionRule$TimeDefinition {
   ZoneOffsetTransitionRule.TimeDefinition STANDARD
   ZoneOffsetTransitionRule.TimeDefinition UTC
   ZoneOffsetTransitionRule.TimeDefinition WALL
@@ -89,6 +89,6 @@ class ZoneOffsetTransitionRule.TimeDefinition -> java.time.zone.ZoneOffsetTransi
 
 #### Exceptions
 
-class ZoneRulesException -> java.time.zone.ZoneRulesException {
+class java.time.zone.ZoneRulesException {
   (String)
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt
index baab868ec0e..708a17a6114 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt
@@ -23,174 +23,174 @@
 #
 #### Interfaces
 
-class BiConsumer -> java.util.function.BiConsumer {
+class java.util.function.BiConsumer {
   void accept(def,def)
   BiConsumer andThen(BiConsumer)
 }
 
-class BiFunction -> java.util.function.BiFunction {
+class java.util.function.BiFunction {
   BiFunction andThen(Function)
   def apply(def,def)
 }
 
-class BinaryOperator -> java.util.function.BinaryOperator {
+class java.util.function.BinaryOperator {
   BinaryOperator maxBy(Comparator)
   BinaryOperator minBy(Comparator)
 }
 
-class BiPredicate -> java.util.function.BiPredicate {
+class java.util.function.BiPredicate {
   BiPredicate and(BiPredicate)
   BiPredicate negate()
   BiPredicate or(BiPredicate)
   boolean test(def,def)
 }
 
-class BooleanSupplier -> java.util.function.BooleanSupplier {
+class java.util.function.BooleanSupplier {
   boolean getAsBoolean()
 }
 
-class Consumer -> java.util.function.Consumer {
+class java.util.function.Consumer {
   void accept(def)
   Consumer andThen(Consumer)
 }
 
-class DoubleBinaryOperator -> java.util.function.DoubleBinaryOperator {
+class java.util.function.DoubleBinaryOperator {
   double applyAsDouble(double,double)
 }
 
-class DoubleConsumer -> java.util.function.DoubleConsumer {
+class java.util.function.DoubleConsumer {
   void accept(double)
   DoubleConsumer andThen(DoubleConsumer)
 }
 
-class DoubleFunction -> java.util.function.DoubleFunction {
+class java.util.function.DoubleFunction {
   def apply(double)
 }
 
-class DoublePredicate -> java.util.function.DoublePredicate {
+class java.util.function.DoublePredicate {
   DoublePredicate and(DoublePredicate)
   DoublePredicate negate()
   DoublePredicate or(DoublePredicate)
   boolean test(double)
 }
 
-class DoubleSupplier -> java.util.function.DoubleSupplier {
+class java.util.function.DoubleSupplier {
   double getAsDouble()
 }
 
-class DoubleToIntFunction -> java.util.function.DoubleToIntFunction {
+class java.util.function.DoubleToIntFunction {
   int applyAsInt(double)
 }
 
-class DoubleToLongFunction -> java.util.function.DoubleToLongFunction {
+class java.util.function.DoubleToLongFunction {
   long applyAsLong(double)
 }
 
-class DoubleUnaryOperator -> java.util.function.DoubleUnaryOperator {
+class java.util.function.DoubleUnaryOperator {
   DoubleUnaryOperator andThen(DoubleUnaryOperator)
   double applyAsDouble(double)
   DoubleUnaryOperator compose(DoubleUnaryOperator)
   DoubleUnaryOperator identity()
 }
 
-class Function -> java.util.function.Function {
+class java.util.function.Function {
   Function andThen(Function)
   def apply(def)
   Function compose(Function)
   Function identity()
 }
 
-class IntBinaryOperator -> java.util.function.IntBinaryOperator {
+class java.util.function.IntBinaryOperator {
   int applyAsInt(int,int)
 }
 
-class IntConsumer -> java.util.function.IntConsumer {
+class java.util.function.IntConsumer {
   void accept(int)
   IntConsumer andThen(IntConsumer)
 }
 
-class IntFunction -> java.util.function.IntFunction {
+class java.util.function.IntFunction {
   def apply(int)
 }
 
-class IntPredicate -> java.util.function.IntPredicate {
+class java.util.function.IntPredicate {
   IntPredicate and(IntPredicate)
   IntPredicate negate()
   IntPredicate or(IntPredicate)
   boolean test(int)
 }
 
-class IntSupplier -> java.util.function.IntSupplier {
+class java.util.function.IntSupplier {
   int getAsInt()
 }
 
-class IntToDoubleFunction -> java.util.function.IntToDoubleFunction {
+class java.util.function.IntToDoubleFunction {
   double applyAsDouble(int)
 }
 
-class IntToLongFunction -> java.util.function.IntToLongFunction {
+class java.util.function.IntToLongFunction {
   long applyAsLong(int)
 }
 
-class IntUnaryOperator -> java.util.function.IntUnaryOperator {
+class java.util.function.IntUnaryOperator {
   IntUnaryOperator andThen(IntUnaryOperator)
   int applyAsInt(int)
   IntUnaryOperator compose(IntUnaryOperator)
   IntUnaryOperator identity()
 }
 
-class LongBinaryOperator -> java.util.function.LongBinaryOperator {
+class java.util.function.LongBinaryOperator {
   long applyAsLong(long,long)
 }
 
-class LongConsumer -> java.util.function.LongConsumer {
+class java.util.function.LongConsumer {
   void accept(long)
   LongConsumer andThen(LongConsumer)
 }
 
-class LongFunction -> java.util.function.LongFunction {
+class java.util.function.LongFunction {
   def apply(long)
 }
 
-class LongPredicate -> java.util.function.LongPredicate {
+class java.util.function.LongPredicate {
   LongPredicate and(LongPredicate)
   LongPredicate negate()
   LongPredicate or(LongPredicate)
   boolean test(long)
 }
 
-class LongSupplier -> java.util.function.LongSupplier {
+class java.util.function.LongSupplier {
   long getAsLong()
 }
 
-class LongToDoubleFunction -> java.util.function.LongToDoubleFunction {
+class java.util.function.LongToDoubleFunction {
   double applyAsDouble(long)
 }
 
-class LongToIntFunction -> java.util.function.LongToIntFunction {
+class java.util.function.LongToIntFunction {
   int applyAsInt(long)
 }
 
-class LongUnaryOperator -> java.util.function.LongUnaryOperator {
+class java.util.function.LongUnaryOperator {
   LongUnaryOperator andThen(LongUnaryOperator)
   long applyAsLong(long)
   LongUnaryOperator compose(LongUnaryOperator)
   LongUnaryOperator identity()
 }
 
-class ObjDoubleConsumer -> java.util.function.ObjDoubleConsumer {
+class java.util.function.ObjDoubleConsumer {
   void accept(def,double)
 }
 
-class ObjIntConsumer -> java.util.function.ObjIntConsumer {
+class java.util.function.ObjIntConsumer {
   void accept(def,int)
 }
 
-class ObjLongConsumer -> java.util.function.ObjLongConsumer {
+class java.util.function.ObjLongConsumer {
   void accept(def,long)
 }
 
-class Predicate -> java.util.function.Predicate {
+class java.util.function.Predicate {
   Predicate and(Predicate)
   Predicate isEqual(def)
   Predicate negate()
@@ -198,34 +198,34 @@ class Predicate -> java.util.function.Predicate {
   boolean test(def)
 }
 
-class Supplier -> java.util.function.Supplier {
+class java.util.function.Supplier {
   def get()
 }
 
-class ToDoubleBiFunction -> java.util.function.ToDoubleBiFunction {
+class java.util.function.ToDoubleBiFunction {
   double applyAsDouble(def,def)
 }
 
-class ToDoubleFunction -> java.util.function.ToDoubleFunction {
+class java.util.function.ToDoubleFunction {
   double applyAsDouble(def)
 }
 
-class ToIntBiFunction -> java.util.function.ToIntBiFunction {
+class java.util.function.ToIntBiFunction {
   int applyAsInt(def,def)
 }
 
-class ToIntFunction -> java.util.function.ToIntFunction {
+class java.util.function.ToIntFunction {
   int applyAsInt(def)
 }
 
-class ToLongBiFunction -> java.util.function.ToLongBiFunction {
+class java.util.function.ToLongBiFunction {
   long applyAsLong(def,def)
 }
 
-class ToLongFunction -> java.util.function.ToLongFunction {
+class java.util.function.ToLongFunction {
   long applyAsLong(def)
 }
 
-class UnaryOperator -> java.util.function.UnaryOperator {
+class java.util.function.UnaryOperator {
   UnaryOperator identity()
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
index 9ea87dd4197..f062d2f6885 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
@@ -22,7 +22,7 @@
 # what methods and fields they have, etc.
 #
 
-class Pattern -> java.util.regex.Pattern {
+class java.util.regex.Pattern {
 # Pattern compile(String)   Intentionally not included. We don't want dynamic patterns because they allow regexes to be generated per time
 #                           the script is run which is super slow. LRegex generates code that calls this method but it skips these checks.
   Predicate asPredicate()
@@ -35,7 +35,7 @@ class Pattern -> java.util.regex.Pattern {
   Stream splitAsStream(CharSequence)
 }
 
-class Matcher -> java.util.regex.Matcher {
+class java.util.regex.Matcher {
   int end()
   int end(int)
   boolean find()
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt
index d531fbb558f..a213300a53f 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt
@@ -24,7 +24,7 @@
 
 #### Interfaces
 
-class BaseStream -> java.util.stream.BaseStream {
+class java.util.stream.BaseStream {
   void close()
   boolean isParallel()
   Iterator iterator()
@@ -33,7 +33,7 @@ class BaseStream -> java.util.stream.BaseStream {
   BaseStream unordered()
 }
 
-class Collector -> java.util.stream.Collector {
+class java.util.stream.Collector {
   BiConsumer accumulator()
   Set characteristics()
   BinaryOperator combiner()
@@ -43,7 +43,7 @@ class Collector -> java.util.stream.Collector {
   Supplier supplier()
 }
 
-class DoubleStream -> java.util.stream.DoubleStream {
+class java.util.stream.DoubleStream {
   boolean allMatch(DoublePredicate)
   boolean anyMatch(DoublePredicate)
   OptionalDouble average()
@@ -82,12 +82,12 @@ class DoubleStream -> java.util.stream.DoubleStream {
   double[] toArray()
 }
 
-class DoubleStream.Builder -> java.util.stream.DoubleStream$Builder {
+class java.util.stream.DoubleStream$Builder {
   DoubleStream.Builder add(double)
   DoubleStream build()
 }
 
-class IntStream -> java.util.stream.IntStream {
+class java.util.stream.IntStream {
   boolean allMatch(IntPredicate)
   boolean anyMatch(IntPredicate)
   DoubleStream asDoubleStream()
@@ -130,12 +130,12 @@ class IntStream -> java.util.stream.IntStream {
   int[] toArray()
 }
 
-class IntStream.Builder -> java.util.stream.IntStream$Builder {
+class java.util.stream.IntStream$Builder {
   IntStream.Builder add(int)
   IntStream build()
 }
 
-class LongStream -> java.util.stream.LongStream {
+class java.util.stream.LongStream {
   boolean allMatch(LongPredicate)
   boolean anyMatch(LongPredicate)
   DoubleStream asDoubleStream()
@@ -177,12 +177,12 @@ class LongStream -> java.util.stream.LongStream {
   long[] toArray()
 }
 
-class LongStream.Builder -> java.util.stream.LongStream$Builder {
+class java.util.stream.LongStream$Builder {
   LongStream.Builder add(long)
   LongStream build()
 }
 
-class Stream -> java.util.stream.Stream {
+class java.util.stream.Stream {
   boolean allMatch(Predicate)
   boolean anyMatch(Predicate)
   Stream.Builder builder()
@@ -221,14 +221,14 @@ class Stream -> java.util.stream.Stream {
   def[] toArray(IntFunction)
 }
 
-class Stream.Builder -> java.util.stream.Stream$Builder {
+class java.util.stream.Stream$Builder {
   Stream.Builder add(def)
   Stream build()
 }
 
 #### Classes
 
-class Collectors -> java.util.stream.Collectors {
+class java.util.stream.Collectors {
   Collector averagingDouble(ToDoubleFunction)
   Collector averagingInt(ToIntFunction)
   Collector averagingLong(ToLongFunction)
@@ -264,7 +264,7 @@ class Collectors -> java.util.stream.Collectors {
 
 #### Enums
 
-class Collector.Characteristics -> java.util.stream.Collector$Characteristics {
+class java.util.stream.Collector$Characteristics {
   Collector.Characteristics CONCURRENT
   Collector.Characteristics IDENTITY_FINISH
   Collector.Characteristics UNORDERED
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
index 164798e68d3..94f302a891d 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
@@ -24,7 +24,7 @@
 
 #### Interfaces
 
-class Collection -> java.util.Collection {
+class java.util.Collection {
   boolean add(def)
   boolean addAll(Collection)
   void clear()
@@ -50,7 +50,7 @@ class Collection -> java.util.Collection {
   List org.elasticsearch.painless.api.Augmentation split(Predicate)
 }
 
-class Comparator -> java.util.Comparator {
+class java.util.Comparator {
   int compare(def,def)
   Comparator comparing(Function)
   Comparator comparing(Function,Comparator)
@@ -70,7 +70,7 @@ class Comparator -> java.util.Comparator {
   Comparator thenComparingLong(ToLongFunction)
 }
 
-class Deque -> java.util.Deque {
+class java.util.Deque {
   void addFirst(def)
   void addLast(def)
   Iterator descendingIterator()
@@ -91,26 +91,26 @@ class Deque -> java.util.Deque {
   boolean removeLastOccurrence(def)
 }
 
-class Enumeration -> java.util.Enumeration {
+class java.util.Enumeration {
   boolean hasMoreElements()
   def nextElement()
 }
 
-class EventListener -> java.util.EventListener {
+class java.util.EventListener {
 }
 
-class Formattable -> java.util.Formattable {
+class java.util.Formattable {
   void formatTo(Formatter,int,int,int)
 }
 
-class Iterator -> java.util.Iterator {
+class java.util.Iterator {
   void forEachRemaining(Consumer)
   boolean hasNext()
   def next()
   void remove()
 }
 
-class List -> java.util.List {
+class java.util.List {
   void add(int,def)
   boolean addAll(int,Collection)
   boolean equals(Object)
@@ -128,7 +128,7 @@ class List -> java.util.List {
   List subList(int,int)
 }
 
-class ListIterator -> java.util.ListIterator {
+class java.util.ListIterator {
   void add(def)
   boolean hasPrevious()
   int nextIndex()
@@ -136,7 +136,7 @@ class ListIterator -> java.util.ListIterator {
   void set(def)
 }
 
-class Map -> java.util.Map {
+class java.util.Map {
   void clear()
   def compute(def,BiFunction)
   def computeIfAbsent(def,Function)
@@ -176,7 +176,7 @@ class Map -> java.util.Map {
   Map org.elasticsearch.painless.api.Augmentation groupBy(BiFunction)
 }
 
-class Map.Entry -> java.util.Map$Entry {
+class java.util.Map$Entry {
   Comparator comparingByKey()
   Comparator comparingByKey(Comparator)
   Comparator comparingByValue()
@@ -188,7 +188,7 @@ class Map.Entry -> java.util.Map$Entry {
   def setValue(def)
 }
 
-class NavigableMap -> java.util.NavigableMap {
+class java.util.NavigableMap {
   Map.Entry ceilingEntry(def)
   def ceilingKey(def)
   NavigableSet descendingKeySet()
@@ -208,7 +208,7 @@ class NavigableMap -> java.util.NavigableMap {
   NavigableMap tailMap(def,boolean)
 }
 
-class NavigableSet -> java.util.NavigableSet {
+class java.util.NavigableSet {
   def ceiling(def)
   Iterator descendingIterator()
   NavigableSet descendingSet()
@@ -222,30 +222,30 @@ class NavigableSet -> java.util.NavigableSet {
   NavigableSet tailSet(def,boolean)
 }
 
-class Observer -> java.util.Observer {
+class java.util.Observer {
   void update(Observable,Object)
 }
 
-class PrimitiveIterator -> java.util.PrimitiveIterator {
+class java.util.PrimitiveIterator {
   void forEachRemaining(def)
 }
 
-class PrimitiveIterator.OfDouble -> java.util.PrimitiveIterator$OfDouble {
+class java.util.PrimitiveIterator$OfDouble {
   Double next()
   double nextDouble()
 }
 
-class PrimitiveIterator.OfInt -> java.util.PrimitiveIterator$OfInt {
+class java.util.PrimitiveIterator$OfInt {
   Integer next()
   int nextInt()
 }
 
-class PrimitiveIterator.OfLong -> java.util.PrimitiveIterator$OfLong {
+class java.util.PrimitiveIterator$OfLong {
   Long next()
   long nextLong()
 }
 
-class Spliterator -> java.util.Spliterator {
+class java.util.Spliterator {
   int CONCURRENT
   int DISTINCT
   int IMMUTABLE
@@ -264,25 +264,25 @@ class Spliterator -> java.util.Spliterator {
   Spliterator trySplit()
 }
 
-class Spliterator.OfPrimitive -> java.util.Spliterator$OfPrimitive {
+class java.util.Spliterator$OfPrimitive {
   void forEachRemaining(def)
   boolean tryAdvance(def)
   Spliterator.OfPrimitive trySplit()
 }
 
-class Spliterator.OfDouble -> java.util.Spliterator$OfDouble {
+class java.util.Spliterator$OfDouble {
   Spliterator.OfDouble trySplit()
 }
 
-class Spliterator.OfInt -> java.util.Spliterator$OfInt {
+class java.util.Spliterator$OfInt {
   Spliterator.OfInt trySplit()
 }
 
-class Spliterator.OfLong -> java.util.Spliterator$OfLong {
+class java.util.Spliterator$OfLong {
   Spliterator.OfLong trySplit()
 }
 
-class Queue -> java.util.Queue {
+class java.util.Queue {
   def element()
   boolean offer(def)
   def peek()
@@ -290,16 +290,16 @@ class Queue -> java.util.Queue {
   def remove()
 }
 
-class RandomAccess -> java.util.RandomAccess {
+class java.util.RandomAccess {
 }
 
-class Set -> java.util.Set {
+class java.util.Set {
   boolean equals(Object)
   int hashCode()
   boolean remove(def)
 }
 
-class SortedMap -> java.util.SortedMap {
+class java.util.SortedMap {
   Comparator comparator()
   def firstKey()
   SortedMap headMap(def)
@@ -308,7 +308,7 @@ class SortedMap -> java.util.SortedMap {
   SortedMap tailMap(def)
 }
 
-class SortedSet -> java.util.SortedSet {
+class java.util.SortedSet {
   Comparator comparator()
   def first()
   SortedSet headSet(def)
@@ -319,55 +319,55 @@ class SortedSet -> java.util.SortedSet {
 
 #### Classes
 
-class AbstractCollection -> java.util.AbstractCollection {
+class java.util.AbstractCollection {
 }
 
-class AbstractList -> java.util.AbstractList {
+class java.util.AbstractList {
 }
 
-class AbstractMap -> java.util.AbstractMap {
+class java.util.AbstractMap {
 }
 
-class AbstractMap.SimpleEntry -> java.util.AbstractMap$SimpleEntry {
+class java.util.AbstractMap$SimpleEntry {
   (def,def)
   (Map.Entry)
 }
 
-class AbstractMap.SimpleImmutableEntry -> java.util.AbstractMap$SimpleImmutableEntry {
+class java.util.AbstractMap$SimpleImmutableEntry {
   (def,def)
   (Map.Entry)
 }
 
-class AbstractQueue -> java.util.AbstractQueue {
+class java.util.AbstractQueue {
 }
 
-class AbstractSequentialList -> java.util.AbstractSequentialList {
+class java.util.AbstractSequentialList {
 }
 
-class AbstractSet -> java.util.AbstractSet {
+class java.util.AbstractSet {
 }
 
-class ArrayDeque -> java.util.ArrayDeque {
+class java.util.ArrayDeque {
   ()
   (Collection)
   ArrayDeque clone()
 }
 
-class ArrayList -> java.util.ArrayList {
+class java.util.ArrayList {
   ()
   (Collection)
   def clone()
   void trimToSize()
 }
 
-class Arrays -> java.util.Arrays {
+class java.util.Arrays {
   List asList(Object[])
   boolean deepEquals(Object[],Object[])
   int deepHashCode(Object[])
   String deepToString(Object[])
 }
 
-class Base64 -> java.util.Base64 {
+class java.util.Base64 {
   Base64.Decoder getDecoder()
   Base64.Encoder getEncoder()
   Base64.Decoder getMimeDecoder()
@@ -377,18 +377,18 @@ class Base64 -> java.util.Base64 {
   Base64.Encoder getUrlEncoder()
 }
 
-class Base64.Decoder -> java.util.Base64$Decoder {
+class java.util.Base64$Decoder {
   int decode(byte[],byte[])
   byte[] decode(String)
 }
 
-class Base64.Encoder -> java.util.Base64$Encoder {
+class java.util.Base64$Encoder {
   int encode(byte[],byte[])
   String encodeToString(byte[])
   Base64.Encoder withoutPadding()
 }
 
-class BitSet -> java.util.BitSet {
+class java.util.BitSet {
   ()
   (int)
   void and(BitSet)
@@ -418,7 +418,7 @@ class BitSet -> java.util.BitSet {
   void xor(BitSet)
 }
 
-class Calendar -> java.util.Calendar {
+class java.util.Calendar {
   int ALL_STYLES
   int AM
   int AM_PM
@@ -516,7 +516,7 @@ class Calendar -> java.util.Calendar {
   Instant toInstant()
 }
 
-class Calendar.Builder -> java.util.Calendar$Builder {
+class java.util.Calendar$Builder {
   ()
   Calendar build()
   Calendar.Builder set(int,int)
@@ -533,7 +533,7 @@ class Calendar.Builder -> java.util.Calendar$Builder {
   Calendar.Builder setWeekDefinition(int,int)
 }
 
-class Collections -> java.util.Collections {
+class java.util.Collections {
   List EMPTY_LIST
   Map EMPTY_MAP
   Set EMPTY_SET
@@ -588,7 +588,7 @@ class Collections -> java.util.Collections {
   SortedSet unmodifiableSortedSet(SortedSet)
 }
 
-class Currency -> java.util.Currency {
+class java.util.Currency {
   Set getAvailableCurrencies()
   String getCurrencyCode()
   int getDefaultFractionDigits()
@@ -600,7 +600,7 @@ class Currency -> java.util.Currency {
   String getSymbol(Locale)
 }
 
-class Date -> java.util.Date {
+class java.util.Date {
   ()
   (long)
   boolean after(Date)
@@ -612,7 +612,7 @@ class Date -> java.util.Date {
   void setTime(long)
 }
 
-class Dictionary -> java.util.Dictionary {
+class java.util.Dictionary {
   Enumeration elements()
   def get(def)
   boolean isEmpty()
@@ -622,7 +622,7 @@ class Dictionary -> java.util.Dictionary {
   int size()
 }
 
-class DoubleSummaryStatistics -> java.util.DoubleSummaryStatistics {
+class java.util.DoubleSummaryStatistics {
   ()
   void combine(DoubleSummaryStatistics)
   double getAverage()
@@ -632,22 +632,22 @@ class DoubleSummaryStatistics -> java.util.DoubleSummaryStatistics {
   double getSum()
 }
 
-class EventListenerProxy -> java.util.EventListenerProxy {
+class java.util.EventListenerProxy {
   EventListener getListener()
 }
 
-class EventObject -> java.util.EventObject {
+class java.util.EventObject {
   (Object)
   Object getSource()
 }
 
-class FormattableFlags -> java.util.FormattableFlags {
+class java.util.FormattableFlags {
   int ALTERNATE
   int LEFT_JUSTIFY
   int UPPERCASE
 }
 
-class Formatter -> java.util.Formatter {
+class java.util.Formatter {
   ()
   (Appendable)
   (Appendable,Locale)
@@ -657,7 +657,7 @@ class Formatter -> java.util.Formatter {
   Appendable out()
 }
 
-class GregorianCalendar -> java.util.GregorianCalendar {
+class java.util.GregorianCalendar {
   int AD
   int BC
   ()
@@ -673,31 +673,31 @@ class GregorianCalendar -> java.util.GregorianCalendar {
   ZonedDateTime toZonedDateTime()
 }
 
-class HashMap -> java.util.HashMap {
+class java.util.HashMap {
   ()
   (Map)
   def clone()
 }
 
-class HashSet -> java.util.HashSet {
+class java.util.HashSet {
   ()
   (Collection)
   def clone()
 }
 
-class Hashtable -> java.util.Hashtable {
+class java.util.Hashtable {
   ()
   (Map)
   def clone()
 }
 
-class IdentityHashMap -> java.util.IdentityHashMap {
+class java.util.IdentityHashMap {
   ()
   (Map)
   def clone()
 }
 
-class IntSummaryStatistics -> java.util.IntSummaryStatistics {
+class java.util.IntSummaryStatistics {
   ()
   void combine(IntSummaryStatistics)
   double getAverage()
@@ -707,23 +707,23 @@ class IntSummaryStatistics -> java.util.IntSummaryStatistics {
   long getSum()
 }
 
-class LinkedHashMap -> java.util.LinkedHashMap {
+class java.util.LinkedHashMap {
   ()
   (Map)
 }
 
-class LinkedHashSet -> java.util.LinkedHashSet {
+class java.util.LinkedHashSet {
   ()
   (Collection)
 }
 
-class LinkedList -> java.util.LinkedList {
+class java.util.LinkedList {
   ()
   (Collection)
   def clone()
 }
 
-class Locale -> java.util.Locale {
+class java.util.Locale {
   Locale CANADA
   Locale CANADA_FRENCH
   Locale CHINA
@@ -788,7 +788,7 @@ class Locale -> java.util.Locale {
   String toLanguageTag()
 }
 
-class Locale.Builder -> java.util.Locale$Builder {
+class java.util.Locale$Builder {
   ()
   Locale.Builder addUnicodeLocaleAttribute(String)
   Locale build()
@@ -805,7 +805,7 @@ class Locale.Builder -> java.util.Locale$Builder {
   Locale.Builder setVariant(String)
 }
 
-class Locale.LanguageRange -> java.util.Locale$LanguageRange {
+class java.util.Locale$LanguageRange {
   double MAX_WEIGHT
   double MIN_WEIGHT
   (String)
@@ -817,7 +817,7 @@ class Locale.LanguageRange -> java.util.Locale$LanguageRange {
   List parse(String,Map)
 }
 
-class LongSummaryStatistics -> java.util.LongSummaryStatistics {
+class java.util.LongSummaryStatistics {
   ()
   void combine(LongSummaryStatistics)
   double getAverage()
@@ -827,7 +827,7 @@ class LongSummaryStatistics -> java.util.LongSummaryStatistics {
   long getSum()
 }
 
-class Objects -> java.util.Objects {
+class java.util.Objects {
   int compare(def,def,Comparator)
   boolean deepEquals(Object,Object)
   boolean equals(Object,Object)
@@ -841,7 +841,7 @@ class Objects -> java.util.Objects {
   String toString(Object,String)
 }
 
-class Observable -> java.util.Observable {
+class java.util.Observable {
   ()
   void addObserver(Observer)
   int countObservers()
@@ -852,7 +852,7 @@ class Observable -> java.util.Observable {
   void notifyObservers(Object)
 }
 
-class Optional -> java.util.Optional {
+class java.util.Optional {
   Optional empty()
   Optional filter(Predicate)
   Optional flatMap(Function)
@@ -867,7 +867,7 @@ class Optional -> java.util.Optional {
   def orElseThrow(Supplier)
 }
 
-class OptionalDouble -> java.util.OptionalDouble {
+class java.util.OptionalDouble {
   OptionalDouble empty()
   double getAsDouble()
   void ifPresent(DoubleConsumer)
@@ -878,7 +878,7 @@ class OptionalDouble -> java.util.OptionalDouble {
   double orElseThrow(Supplier)
 }
 
-class OptionalInt -> java.util.OptionalInt {
+class java.util.OptionalInt {
   OptionalInt empty()
   int getAsInt()
   void ifPresent(IntConsumer)
@@ -889,7 +889,7 @@ class OptionalInt -> java.util.OptionalInt {
   int orElseThrow(Supplier)
 }
 
-class OptionalLong -> java.util.OptionalLong {
+class java.util.OptionalLong {
   OptionalLong empty()
   long getAsLong()
   void ifPresent(LongConsumer)
@@ -900,12 +900,12 @@ class OptionalLong -> java.util.OptionalLong {
   long orElseThrow(Supplier)
 }
 
-class PriorityQueue -> java.util.PriorityQueue {
+class java.util.PriorityQueue {
   ()
   (Comparator)
 }
 
-class Random -> java.util.Random {
+class java.util.Random {
   ()
   (long)
   DoubleStream doubles(long)
@@ -925,7 +925,7 @@ class Random -> java.util.Random {
   void setSeed(long)
 }
 
-class SimpleTimeZone -> java.util.SimpleTimeZone {
+class java.util.SimpleTimeZone {
   int STANDARD_TIME
   int UTC_TIME
   int WALL_TIME
@@ -944,7 +944,7 @@ class SimpleTimeZone -> java.util.SimpleTimeZone {
   void setStartYear(int)
 }
 
-class Spliterators -> java.util.Spliterators {
+class java.util.Spliterators {
   Spliterator.OfDouble emptyDoubleSpliterator()
   Spliterator.OfInt emptyIntSpliterator()
   Spliterator.OfLong emptyLongSpliterator()
@@ -955,7 +955,7 @@ class Spliterators -> java.util.Spliterators {
   Spliterator spliteratorUnknownSize(Iterator,int)
 }
 
-class Stack -> java.util.Stack {
+class java.util.Stack {
   ()
   def push(def)
   def pop()
@@ -964,7 +964,7 @@ class Stack -> java.util.Stack {
   int search(def)
 }
 
-class StringJoiner -> java.util.StringJoiner {
+class java.util.StringJoiner {
   (CharSequence)
   (CharSequence,CharSequence,CharSequence)
   StringJoiner add(CharSequence)
@@ -973,7 +973,7 @@ class StringJoiner -> java.util.StringJoiner {
   StringJoiner setEmptyValue(CharSequence)
 }
 
-class StringTokenizer -> java.util.StringTokenizer {
+class java.util.StringTokenizer {
   (String)
   (String,String)
   (String,String,boolean)
@@ -983,7 +983,7 @@ class StringTokenizer -> java.util.StringTokenizer {
   String nextToken(String)
 }
 
-class TimeZone -> java.util.TimeZone {
+class java.util.TimeZone {
   int LONG
   int SHORT
   def clone()
@@ -1008,19 +1008,19 @@ class TimeZone -> java.util.TimeZone {
   boolean useDaylightTime()
 }
 
-class TreeMap -> java.util.TreeMap {
+class java.util.TreeMap {
   ()
   (Comparator)
   def clone()
 }
 
-class TreeSet -> java.util.TreeSet {
+class java.util.TreeSet {
   ()
   (Comparator)
   def clone()
 }
 
-class UUID -> java.util.UUID {
+class java.util.UUID {
   (long,long)
   int compareTo(UUID)
   int clockSequence()
@@ -1034,7 +1034,7 @@ class UUID -> java.util.UUID {
   int version()
 }
 
-class Vector -> java.util.Vector {
+class java.util.Vector {
   ()
   (Collection)
   void addElement(def)
@@ -1054,19 +1054,19 @@ class Vector -> java.util.Vector {
 
 #### Enums
 
-class Formatter.BigDecimalLayoutForm -> java.util.Formatter$BigDecimalLayoutForm {
+class java.util.Formatter$BigDecimalLayoutForm {
   Formatter.BigDecimalLayoutForm DECIMAL_FLOAT
   Formatter.BigDecimalLayoutForm SCIENTIFIC
 }
 
-class Locale.Category -> java.util.Locale$Category {
+class java.util.Locale$Category {
   Locale.Category DISPLAY
   Locale.Category FORMAT
   Locale.Category valueOf(String)
   Locale.Category[] values()
 }
 
-class Locale.FilteringMode -> java.util.Locale$FilteringMode {
+class java.util.Locale$FilteringMode {
   Locale.FilteringMode AUTOSELECT_FILTERING
   Locale.FilteringMode EXTENDED_FILTERING
   Locale.FilteringMode IGNORE_EXTENDED_RANGES
@@ -1078,101 +1078,101 @@ class Locale.FilteringMode -> java.util.Locale$FilteringMode {
 
 #### Exceptions
 
-class ConcurrentModificationException -> java.util.ConcurrentModificationException {
+class java.util.ConcurrentModificationException {
   ()
   (String)
 }
 
-class DuplicateFormatFlagsException -> java.util.DuplicateFormatFlagsException {
+class java.util.DuplicateFormatFlagsException {
   (String)
   String getFlags()
 }
 
-class EmptyStackException -> java.util.EmptyStackException {
+class java.util.EmptyStackException {
   ()
 }
 
-class FormatFlagsConversionMismatchException -> java.util.FormatFlagsConversionMismatchException {
+class java.util.FormatFlagsConversionMismatchException {
   (String,char)
   char getConversion()
   String getFlags()
 }
 
-class FormatterClosedException -> java.util.FormatterClosedException {
+class java.util.FormatterClosedException {
   ()
 }
 
-class IllegalFormatCodePointException -> java.util.IllegalFormatCodePointException {
+class java.util.IllegalFormatCodePointException {
   (int)
   int getCodePoint()
 }
 
-class IllegalFormatConversionException -> java.util.IllegalFormatConversionException {
+class java.util.IllegalFormatConversionException {
   char getConversion()
 }
 
-class IllegalFormatException -> java.util.IllegalFormatException {
+class java.util.IllegalFormatException {
 }
 
-class IllegalFormatFlagsException -> java.util.IllegalFormatFlagsException {
+class java.util.IllegalFormatFlagsException {
   (String)
   String getFlags()
 }
 
-class IllegalFormatPrecisionException -> java.util.IllegalFormatPrecisionException {
+class java.util.IllegalFormatPrecisionException {
   (int)
   int getPrecision()
 }
 
-class IllegalFormatWidthException -> java.util.IllegalFormatWidthException {
+class java.util.IllegalFormatWidthException {
   (int)
   int getWidth()
 }
 
-class IllformedLocaleException -> java.util.IllformedLocaleException {
+class java.util.IllformedLocaleException {
   ()
   (String)
   (String,int)
   int getErrorIndex()
 }
 
-class InputMismatchException -> java.util.InputMismatchException {
+class java.util.InputMismatchException {
   ()
   (String)
 }
 
-class MissingFormatArgumentException -> java.util.MissingFormatArgumentException {
+class java.util.MissingFormatArgumentException {
   (String)
   String getFormatSpecifier()
 }
 
-class MissingFormatWidthException -> java.util.MissingFormatWidthException {
+class java.util.MissingFormatWidthException {
   (String)
   String getFormatSpecifier()
 }
 
-class MissingResourceException -> java.util.MissingResourceException {
+class java.util.MissingResourceException {
   (String,String,String)
   String getClassName()
   String getKey()
 }
 
-class NoSuchElementException -> java.util.NoSuchElementException {
+class java.util.NoSuchElementException {
   ()
   (String)
 }
 
-class TooManyListenersException -> java.util.TooManyListenersException {
+class java.util.TooManyListenersException {
   ()
   (String)
 }
 
-class UnknownFormatConversionException -> java.util.UnknownFormatConversionException {
+class java.util.UnknownFormatConversionException {
   (String)
   String getConversion()
 }
 
-class UnknownFormatFlagsException -> java.util.UnknownFormatFlagsException {
+class java.util.UnknownFormatFlagsException {
   (String)
   String getFlags()
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/joda.time.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/joda.time.txt
index 6899e287868..3b2f379c38e 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/joda.time.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/joda.time.txt
@@ -26,7 +26,7 @@
 # convenient access via the scripting API. classes are fully qualified to avoid
 # any confusion with java.time
 
-class org.joda.time.ReadableInstant -> org.joda.time.ReadableInstant {
+class org.joda.time.ReadableInstant {
   boolean equals(Object)
   long getMillis()
   int hashCode()
@@ -36,7 +36,7 @@ class org.joda.time.ReadableInstant -> org.joda.time.ReadableInstant {
   String toString()
 }
 
-class org.joda.time.ReadableDateTime -> org.joda.time.ReadableDateTime {
+class org.joda.time.ReadableDateTime {
   int getCenturyOfEra()
   int getDayOfMonth()
   int getDayOfWeek()
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
index 5ff486b0b81..7c08f76f452 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
@@ -24,53 +24,53 @@
 
 #### Primitive types
 
-class void -> void {
+class void only_fqn {
 }
 
-class boolean -> boolean {
+class boolean only_fqn {
 }
 
-class byte -> byte {
+class byte only_fqn {
 }
 
-class short -> short {
+class short only_fqn {
 }
 
-class char -> char {
+class char only_fqn {
 }
 
-class int -> int {
+class int only_fqn {
 }
 
-class long -> long {
+class long only_fqn {
 }
 
-class float -> float {
+class float only_fqn {
 }
 
-class double -> double {
+class double only_fqn {
 }
 
 #### Painless debugging API
 
-class Debug -> org.elasticsearch.painless.api.Debug {
+class org.elasticsearch.painless.api.Debug {
   void explain(Object)
 }
 
 #### ES Scripting API
 
-class org.elasticsearch.common.geo.GeoPoint -> org.elasticsearch.common.geo.GeoPoint {
+class org.elasticsearch.common.geo.GeoPoint {
   double getLat()
   double getLon()
 }
 
-class org.elasticsearch.index.fielddata.ScriptDocValues.Strings -> org.elasticsearch.index.fielddata.ScriptDocValues$Strings {
+class org.elasticsearch.index.fielddata.ScriptDocValues$Strings {
   String get(int)
   String getValue()
   List getValues()
 }
 
-class org.elasticsearch.index.fielddata.ScriptDocValues.Longs -> org.elasticsearch.index.fielddata.ScriptDocValues$Longs {
+class org.elasticsearch.index.fielddata.ScriptDocValues$Longs {
   Long get(int)
   long getValue()
   List getValues()
@@ -78,7 +78,7 @@ class org.elasticsearch.index.fielddata.ScriptDocValues.Longs -> org.elasticsear
   List getDates()
 }
 
-class org.elasticsearch.index.fielddata.ScriptDocValues.Dates -> org.elasticsearch.index.fielddata.ScriptDocValues$Dates {
+class org.elasticsearch.index.fielddata.ScriptDocValues$Dates {
   org.joda.time.ReadableDateTime get(int)
   org.joda.time.ReadableDateTime getValue()
   List getValues()
@@ -86,13 +86,13 @@ class org.elasticsearch.index.fielddata.ScriptDocValues.Dates -> org.elasticsear
   List getDates()
 }
 
-class org.elasticsearch.index.fielddata.ScriptDocValues.Doubles -> org.elasticsearch.index.fielddata.ScriptDocValues$Doubles {
+class org.elasticsearch.index.fielddata.ScriptDocValues$Doubles {
   Double get(int)
   double getValue()
   List getValues()
 }
 
-class org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints -> org.elasticsearch.index.fielddata.ScriptDocValues$GeoPoints {
+class org.elasticsearch.index.fielddata.ScriptDocValues$GeoPoints {
   org.elasticsearch.common.geo.GeoPoint get(int)
   org.elasticsearch.common.geo.GeoPoint getValue()
   List getValues()
@@ -110,19 +110,19 @@ class org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints -> org.elastic
   double geohashDistanceWithDefault(String,double)
 }
 
-class org.elasticsearch.index.fielddata.ScriptDocValues.Booleans -> org.elasticsearch.index.fielddata.ScriptDocValues$Booleans {
+class org.elasticsearch.index.fielddata.ScriptDocValues$Booleans {
   Boolean get(int)
   boolean getValue()
   List getValues()
 }
 
-class org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs -> org.elasticsearch.index.fielddata.ScriptDocValues$BytesRefs {
+class org.elasticsearch.index.fielddata.ScriptDocValues$BytesRefs {
   BytesRef get(int)
   BytesRef getValue()
   List getValues()
 }
 
-class BytesRef -> org.apache.lucene.util.BytesRef {
+class org.apache.lucene.util.BytesRef {
   byte[] bytes
   int offset
   int length
@@ -130,7 +130,7 @@ class BytesRef -> org.apache.lucene.util.BytesRef {
   String utf8ToString()
 }
 
-class org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues -> org.elasticsearch.index.mapper.IpFieldMapper$IpFieldType$IpScriptDocValues {
+class org.elasticsearch.index.mapper.IpFieldMapper$IpFieldType$IpScriptDocValues {
   String get(int)
   String getValue()
   List getValues()
@@ -138,7 +138,7 @@ class org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues
 
 # for testing.
 # currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods
-class org.elasticsearch.painless.FeatureTest -> org.elasticsearch.painless.FeatureTest {
+class org.elasticsearch.painless.FeatureTest only_fqn {
   ()
   (int,int)
   int getX()
@@ -153,28 +153,28 @@ class org.elasticsearch.painless.FeatureTest -> org.elasticsearch.painless.Featu
   int org.elasticsearch.painless.FeatureTestAugmentation addToTotal(int)
 }
 
-class org.elasticsearch.search.lookup.FieldLookup -> org.elasticsearch.search.lookup.FieldLookup {
+class org.elasticsearch.search.lookup.FieldLookup {
   def getValue()
   List getValues()
   boolean isEmpty()
 }
 
-class org.elasticsearch.index.similarity.ScriptedSimilarity.Query -> org.elasticsearch.index.similarity.ScriptedSimilarity$Query {
+class org.elasticsearch.index.similarity.ScriptedSimilarity$Query {
   float getBoost()
 }
 
-class org.elasticsearch.index.similarity.ScriptedSimilarity.Field -> org.elasticsearch.index.similarity.ScriptedSimilarity$Field {
+class org.elasticsearch.index.similarity.ScriptedSimilarity$Field {
   long getDocCount()
   long getSumDocFreq()
   long getSumTotalTermFreq()
 }
 
-class org.elasticsearch.index.similarity.ScriptedSimilarity.Term -> org.elasticsearch.index.similarity.ScriptedSimilarity$Term {
+class org.elasticsearch.index.similarity.ScriptedSimilarity$Term {
   long getDocFreq()
   long getTotalTermFreq()
 }
 
-class org.elasticsearch.index.similarity.ScriptedSimilarity.Doc -> org.elasticsearch.index.similarity.ScriptedSimilarity$Doc {
+class org.elasticsearch.index.similarity.ScriptedSimilarity$Doc {
   int getLength()
   float getFreq()
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
index fe50578df1a..ba31ea48d92 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
@@ -44,7 +44,7 @@ public class DebugTests extends ScriptTestCase {
         assertSame(dummy, e.getObjectToExplain());
         assertThat(e.getHeaders(definition), hasEntry("es.to_string", singletonList(dummy.toString())));
         assertThat(e.getHeaders(definition), hasEntry("es.java_class", singletonList("java.lang.Object")));
-        assertThat(e.getHeaders(definition), hasEntry("es.painless_class", singletonList("Object")));
+        assertThat(e.getHeaders(definition), hasEntry("es.painless_class", singletonList("java.lang.Object")));
 
         // Null should be ok
         e = expectScriptThrows(PainlessExplainError.class, () -> exec("Debug.explain(null)"));
@@ -71,7 +71,7 @@ public class DebugTests extends ScriptTestCase {
         ScriptException e = expectThrows(ScriptException.class, () -> exec("Debug.explain(params.a)", params, true));
         assertEquals(singletonList("jumped over the moon"), e.getMetadata("es.to_string"));
         assertEquals(singletonList("java.lang.String"), e.getMetadata("es.java_class"));
-        assertEquals(singletonList("String"), e.getMetadata("es.painless_class"));
+        assertEquals(singletonList("java.lang.String"), e.getMetadata("es.painless_class"));
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeException(e);
@@ -79,7 +79,7 @@ public class DebugTests extends ScriptTestCase {
                 ElasticsearchException read = (ScriptException) in.readException();
                 assertEquals(singletonList("jumped over the moon"), read.getMetadata("es.to_string"));
                 assertEquals(singletonList("java.lang.String"), read.getMetadata("es.java_class"));
-                assertEquals(singletonList("String"), read.getMetadata("es.painless_class"));
+                assertEquals(singletonList("java.lang.String"), read.getMetadata("es.painless_class"));
             }
         }
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
index 92ff9ef3c93..0a66b67a2e8 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
@@ -153,7 +153,7 @@ public class RegexTests extends ScriptTestCase {
     }
 
     public void testSplitAsStream() {
-        assertEquals(new HashSet<>(Arrays.asList("cat", "dog")), exec("/,/.splitAsStream('cat,dog').collect(Collectors.toSet())"));
+        assertEquals(new HashSet(Arrays.asList("cat", "dog")), exec("/,/.splitAsStream('cat,dog').collect(Collectors.toSet())"));
     }
 
     // Make sure the flags are set
@@ -252,7 +252,7 @@ public class RegexTests extends ScriptTestCase {
         IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("Pattern.compile('aa')");
         });
-        assertEquals("Unknown call [compile] with [1] arguments on type [Pattern].", e.getMessage());
+        assertEquals("Unknown call [compile] with [1] arguments on type [java.util.regex.Pattern].", e.getMessage());
     }
 
     public void testBadRegexPattern() {
@@ -271,7 +271,7 @@ public class RegexTests extends ScriptTestCase {
         ClassCastException e = expectScriptThrows(ClassCastException.class, () -> {
             exec("12 ==~ /cat/");
         });
-        assertEquals("Cannot cast from [int] to [String].", e.getMessage());
+        assertEquals("Cannot cast from [int] to [java.lang.String].", e.getMessage());
     }
 
     public void testBogusRegexFlag() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
index b3842615859..a18c27cdef5 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
@@ -162,12 +162,13 @@ public class NodeToStringTests extends ESTestCase {
         Location l = new Location(getTestName(), 0);
         AExpression child = new EConstant(l, "test");
         Cast cast = new Cast(Definition.DEFINITION.StringType, Definition.DEFINITION.IntegerType, true);
-        assertEquals("(ECast Integer (EConstant String 'test'))", new ECast(l, child, cast).toString());
+        assertEquals("(ECast java.lang.Integer (EConstant String 'test'))", new ECast(l, child, cast).toString());
 
         l = new Location(getTestName(), 1);
         child = new EBinary(l, Operation.ADD, new EConstant(l, "test"), new EConstant(l, 12));
         cast = new Cast(Definition.DEFINITION.IntegerType, Definition.DEFINITION.BooleanType, true);
-        assertEquals("(ECast Boolean (EBinary (EConstant String 'test') + (EConstant Integer 12)))", new ECast(l, child, cast).toString());
+        assertEquals("(ECast java.lang.Boolean (EBinary (EConstant String 'test') + (EConstant Integer 12)))",
+            new ECast(l, child, cast).toString());
     }
 
     public void testEComp() {

From f27cb96a64ca515eeb2608c79edc58e360bc07e8 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux 
Date: Tue, 12 Dec 2017 09:30:36 +0100
Subject: [PATCH 243/297] Use AmazonS3.doesObjectExist() method in
 S3BlobContainer (#27723)

This pull request changes the S3BlobContainer.blobExists() method implementation
to make it use the AmazonS3.doesObjectExist() method instead of
AmazonS3.getObjectMetadata(). The AmazonS3 implementation takes care of
catching any thrown AmazonS3Exception and compares its response code with 404,
returning false (object does not exist) or lets the exception be propagated.
---
 .../blobstore/BlobStoreRepository.java        | 50 +++++++----------
 .../snapshots/SnapshotShardsService.java      |  1 -
 .../snapshots/SnapshotsService.java           | 56 +++++++++----------
 .../repositories/s3/S3BlobContainer.java      | 13 ++---
 .../repositories/s3/MockAmazonS3.java         |  9 ++-
 5 files changed, 58 insertions(+), 71 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index a66a5a51d10..9afbb528782 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -368,9 +368,9 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             writeIndexGen(updatedRepositoryData, repositoryStateId);
 
             // delete the snapshot file
-            safeSnapshotBlobDelete(snapshot, snapshotId.getUUID());
+            deleteSnapshotBlobIgnoringErrors(snapshot, snapshotId.getUUID());
             // delete the global metadata file
-            safeGlobalMetaDataBlobDelete(snapshot, snapshotId.getUUID());
+            deleteGlobalMetaDataBlobIgnoringErrors(snapshot, snapshotId.getUUID());
 
             // Now delete all indices
             for (String index : indices) {
@@ -422,37 +422,27 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         }
     }
 
-    private void safeSnapshotBlobDelete(final SnapshotInfo snapshotInfo, final String blobId) {
-        if (snapshotInfo != null) {
-            // we know the version the snapshot was created with
-            try {
-                snapshotFormat.delete(snapshotsBlobContainer, blobId);
-            } catch (IOException e) {
-                logger.warn((Supplier) () -> new ParameterizedMessage("[{}] Unable to delete snapshot file [{}]", snapshotInfo.snapshotId(), blobId), e);
-            }
-        } else {
-            try {
-                snapshotFormat.delete(snapshotsBlobContainer, blobId);
-            } catch (IOException e) {
-                // snapshot file could not be deleted, log the error
+    private void deleteSnapshotBlobIgnoringErrors(final SnapshotInfo snapshotInfo, final String blobId) {
+        try {
+            snapshotFormat.delete(snapshotsBlobContainer, blobId);
+        } catch (IOException e) {
+            if (snapshotInfo != null) {
+                logger.warn((Supplier) () -> new ParameterizedMessage("[{}] Unable to delete snapshot file [{}]",
+                    snapshotInfo.snapshotId(), blobId), e);
+            } else {
                 logger.warn((Supplier) () -> new ParameterizedMessage("Unable to delete snapshot file [{}]", blobId), e);
             }
         }
     }
 
-    private void safeGlobalMetaDataBlobDelete(final SnapshotInfo snapshotInfo, final String blobId) {
-        if (snapshotInfo != null) {
-            // we know the version the snapshot was created with
-            try {
-                globalMetaDataFormat.delete(snapshotsBlobContainer, blobId);
-            } catch (IOException e) {
-                logger.warn((Supplier) () -> new ParameterizedMessage("[{}] Unable to delete global metadata file [{}]", snapshotInfo.snapshotId(), blobId), e);
-            }
-        } else {
-            try {
-                globalMetaDataFormat.delete(snapshotsBlobContainer, blobId);
-            } catch (IOException e) {
-                // global metadata file could not be deleted, log the error
+    private void deleteGlobalMetaDataBlobIgnoringErrors(final SnapshotInfo snapshotInfo, final String blobId) {
+        try {
+            globalMetaDataFormat.delete(snapshotsBlobContainer, blobId);
+        } catch (IOException e) {
+            if (snapshotInfo != null) {
+                logger.warn((Supplier) () -> new ParameterizedMessage("[{}] Unable to delete global metadata file [{}]",
+                    snapshotInfo.snapshotId(), blobId), e);
+            } else {
                 logger.warn((Supplier) () -> new ParameterizedMessage("Unable to delete global metadata file [{}]", blobId), e);
             }
         }
@@ -512,9 +502,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             // When we delete corrupted snapshots we might not know which version we are dealing with
             // We can try detecting the version based on the metadata file format
             assert ignoreIndexErrors;
-            if (globalMetaDataFormat.exists(snapshotsBlobContainer, snapshotId.getUUID())) {
-                snapshotVersion = Version.CURRENT;
-            } else {
+            if (globalMetaDataFormat.exists(snapshotsBlobContainer, snapshotId.getUUID()) == false) {
                 throw new SnapshotMissingException(metadata.name(), snapshotId);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index 15f70e8b2c6..d3e0aac5e2a 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -581,7 +581,6 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
                             entries.add(updatedEntry);
                             // Finalize snapshot in the repository
                             snapshotsService.endSnapshot(updatedEntry);
-                            logger.info("snapshot [{}] is done", updatedEntry.snapshot());
                         }
                     } else {
                         entries.add(entry);
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 5092a58adaa..7a5fdaa7052 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -263,7 +263,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                                                                 null);
                     snapshots = new SnapshotsInProgress(newSnapshot);
                 } else {
-                    throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, "a snapshot is already running");
+                    throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, " a snapshot is already running");
                 }
                 return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build();
             }
@@ -363,6 +363,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
 
             repository.initializeSnapshot(snapshot.snapshot().getSnapshotId(), snapshot.indices(), metaData);
             snapshotCreated = true;
+
+            logger.info("snapshot [{}] started", snapshot.snapshot());
             if (snapshot.indices().isEmpty()) {
                 // No indices in this snapshot - we are done
                 userCreateSnapshotListener.onResponse();
@@ -947,35 +949,33 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
      * @param failure failure reason or null if snapshot was successful
      */
     private void endSnapshot(final SnapshotsInProgress.Entry entry, final String failure) {
-        threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
-            @Override
-            public void run() {
-                final Snapshot snapshot = entry.snapshot();
-                try {
-                    final Repository repository = repositoriesService.repository(snapshot.getRepository());
-                    logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshot, entry.state(), failure);
-                    ArrayList shardFailures = new ArrayList<>();
-                    for (ObjectObjectCursor shardStatus : entry.shards()) {
-                        ShardId shardId = shardStatus.key;
-                        ShardSnapshotStatus status = shardStatus.value;
-                        if (status.state().failed()) {
-                            shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, status.reason()));
-                        }
+        threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
+            final Snapshot snapshot = entry.snapshot();
+            try {
+                final Repository repository = repositoriesService.repository(snapshot.getRepository());
+                logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshot, entry.state(), failure);
+                ArrayList shardFailures = new ArrayList<>();
+                for (ObjectObjectCursor shardStatus : entry.shards()) {
+                    ShardId shardId = shardStatus.key;
+                    ShardSnapshotStatus status = shardStatus.value;
+                    if (status.state().failed()) {
+                        shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, status.reason()));
                     }
-                    SnapshotInfo snapshotInfo = repository.finalizeSnapshot(
-                        snapshot.getSnapshotId(),
-                        entry.indices(),
-                        entry.startTime(),
-                        failure,
-                        entry.shards().size(),
-                        Collections.unmodifiableList(shardFailures),
-                        entry.getRepositoryStateId(),
-                        entry.includeGlobalState());
-                    removeSnapshotFromClusterState(snapshot, snapshotInfo, null);
-                } catch (Exception e) {
-                    logger.warn((Supplier) () -> new ParameterizedMessage("[{}] failed to finalize snapshot", snapshot), e);
-                    removeSnapshotFromClusterState(snapshot, null, e);
                 }
+                SnapshotInfo snapshotInfo = repository.finalizeSnapshot(
+                    snapshot.getSnapshotId(),
+                    entry.indices(),
+                    entry.startTime(),
+                    failure,
+                    entry.shards().size(),
+                    Collections.unmodifiableList(shardFailures),
+                    entry.getRepositoryStateId(),
+                    entry.includeGlobalState());
+                removeSnapshotFromClusterState(snapshot, snapshotInfo, null);
+                logger.info("snapshot [{}] completed with state [{}]", snapshot, snapshotInfo.state());
+            } catch (Exception e) {
+                logger.warn((Supplier) () -> new ParameterizedMessage("[{}] failed to finalize snapshot", snapshot), e);
+                removeSnapshotFromClusterState(snapshot, null, e);
             }
         });
     }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
index bb1130db42d..401ef0933a8 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
@@ -73,14 +73,9 @@ class S3BlobContainer extends AbstractBlobContainer {
     @Override
     public boolean blobExists(String blobName) {
         try {
-            return SocketAccess.doPrivileged(() -> {
-                blobStore.client().getObjectMetadata(blobStore.bucket(), buildKey(blobName));
-                return true;
-            });
-        } catch (AmazonS3Exception e) {
-            return false;
+            return SocketAccess.doPrivileged(() -> blobStore.client().doesObjectExist(blobStore.bucket(), buildKey(blobName)));
         } catch (Exception e) {
-            throw new BlobStoreException("failed to check if blob exists", e);
+            throw new BlobStoreException("Failed to check if blob [" + blobName +"] exists", e);
         }
     }
 
@@ -102,7 +97,7 @@ class S3BlobContainer extends AbstractBlobContainer {
     @Override
     public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
         if (blobExists(blobName)) {
-            throw new FileAlreadyExistsException("blob [" + blobName + "] already exists, cannot overwrite");
+            throw new FileAlreadyExistsException("Blob [" + blobName + "] already exists, cannot overwrite");
         }
 
         SocketAccess.doPrivilegedIOException(() -> {
@@ -117,7 +112,7 @@ class S3BlobContainer extends AbstractBlobContainer {
 
     @Override
     public void deleteBlob(String blobName) throws IOException {
-        if (!blobExists(blobName)) {
+        if (blobExists(blobName) == false) {
             throw new NoSuchFileException("Blob [" + blobName + "] does not exist");
         }
 
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java
index af7638b4111..a090fdd5281 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java
@@ -21,6 +21,7 @@ package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkClientException;
 import com.amazonaws.services.s3.AbstractAmazonS3;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.CopyObjectRequest;
@@ -36,14 +37,12 @@ import com.amazonaws.services.s3.model.PutObjectResult;
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectInputStream;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.util.Base64;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
 import java.net.InetAddress;
 import java.net.Socket;
-import java.security.DigestInputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -88,6 +87,12 @@ class MockAmazonS3 extends AbstractAmazonS3 {
         return true;
     }
 
+    @Override
+    public boolean doesObjectExist(String bucketName, String objectName) throws AmazonServiceException, SdkClientException {
+        simulateS3SocketConnection();
+        return blobs.containsKey(objectName);
+    }
+
     @Override
     public ObjectMetadata getObjectMetadata(
             GetObjectMetadataRequest getObjectMetadataRequest)

From cfc3b2d34473312dd89c339920d88d143e8fb8c9 Mon Sep 17 00:00:00 2001
From: Boaz Leskes 
Date: Tue, 12 Dec 2017 09:38:54 +0100
Subject: [PATCH 244/297] remove
 InternalEngine.compareOpToLuceneDocBasedOnVersions as it is unused

relates #27720
---
 .../elasticsearch/index/engine/InternalEngine.java  | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index ecf589cbaf9..47056c3b010 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -648,19 +648,6 @@ public class InternalEngine extends Engine {
         return versionValue;
     }
 
-    private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnVersions(final Operation op)
-        throws IOException {
-        assert op.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO : "op is resolved based on versions but have a seq#";
-        assert op.version() >= 0 : "versions should be non-negative. got " + op.version();
-        final VersionValue versionValue = resolveDocVersion(op);
-        if (versionValue == null) {
-            return OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND;
-        } else {
-            return op.versionType().isVersionConflictForWrites(versionValue.version, op.version(), versionValue.isDelete()) ?
-                OpVsLuceneDocStatus.OP_STALE_OR_EQUAL : OpVsLuceneDocStatus.OP_NEWER;
-        }
-    }
-
     private boolean canOptimizeAddDocument(Index index) {
         if (index.getAutoGeneratedIdTimestamp() != IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP) {
             assert index.getAutoGeneratedIdTimestamp() >= 0 : "autoGeneratedIdTimestamp must be positive but was: "

From a1ed3471103ef0f230c4fb2721ef9dbb901fcc2b Mon Sep 17 00:00:00 2001
From: Tanguy Leroux 
Date: Tue, 12 Dec 2017 09:51:18 +0100
Subject: [PATCH 245/297] Fail restore when the shard allocations max retries
 count is reached (#27493)

This commit changes the RestoreService so that it now fails the snapshot
restore if one of the shards to restore has failed to be allocated. It also adds
a new RestoreInProgressAllocationDecider that forbids such shards to be
allocated again. This way, when a restore is impossible or failed too many
times, the user is forced to take a manual action (like deleting the index
which failed shards) in order to try to restore it again.

This behaviour has been implemented because when the allocation of a
shard has been retried too many times, the MaxRetryDecider is engaged
to prevent any future allocation of the failed shard. If it happens while
restoring a snapshot, the restore hanged and was never completed because
it stayed around waiting for the shards to be assigned (and that won't happen).
It also blocked future attempts to restore the snapshot again. With this commit,
the restore does not hang and is marked as failed, leaving failed shards
around for investigation.

This is the second part of the #26865 issue.

Closes #26865
---
 .../elasticsearch/cluster/ClusterModule.java  |   2 +
 .../routing/allocation/AllocationService.java |   9 +-
 .../RestoreInProgressAllocationDecider.java   |  86 ++++++++
 .../snapshots/RestoreService.java             |  37 ++--
 .../cluster/ClusterModuleTests.java           |   2 +
 .../allocation/ThrottlingAllocationTests.java |  38 +++-
 ...storeInProgressAllocationDeciderTests.java | 208 ++++++++++++++++++
 .../SharedClusterSnapshotRestoreIT.java       | 188 +++++++++++++++-
 8 files changed, 541 insertions(+), 29 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java
 create mode 100644 core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java

diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index a4bb6a55925..9baa47fbc26 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -54,6 +54,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.ResizeAllocationDeci
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.RestoreInProgressAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
@@ -191,6 +192,7 @@ public class ClusterModule extends AbstractModule {
         addAllocationDecider(deciders, new EnableAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new NodeVersionAllocationDecider(settings));
         addAllocationDecider(deciders, new SnapshotInProgressAllocationDecider(settings));
+        addAllocationDecider(deciders, new RestoreInProgressAllocationDecider(settings));
         addAllocationDecider(deciders, new FilterAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new SameShardAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new DiskThresholdDecider(settings, clusterSettings));
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index 774e4b9301c..d79237b8a65 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -48,6 +48,8 @@ import java.util.List;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
 
 
@@ -135,13 +137,14 @@ public class AllocationService extends AbstractComponent {
         return newState;
     }
 
+    // Used for testing
     public ClusterState applyFailedShard(ClusterState clusterState, ShardRouting failedShard) {
-        return applyFailedShards(clusterState, Collections.singletonList(new FailedShard(failedShard, null, null)),
-            Collections.emptyList());
+        return applyFailedShards(clusterState, singletonList(new FailedShard(failedShard, null, null)), emptyList());
     }
 
+    // Used for testing
     public ClusterState applyFailedShards(ClusterState clusterState, List failedShards) {
-        return applyFailedShards(clusterState, failedShards, Collections.emptyList());
+        return applyFailedShards(clusterState, failedShards, emptyList());
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java
new file mode 100644
index 00000000000..3fefd4e0abb
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation.decider;
+
+import org.elasticsearch.cluster.RestoreInProgress;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.snapshots.Snapshot;
+
+/**
+ * This {@link AllocationDecider} prevents shards that have failed to be
+ * restored from a snapshot to be allocated.
+ */
+public class RestoreInProgressAllocationDecider extends AllocationDecider {
+
+    public static final String NAME = "restore_in_progress";
+
+    /**
+     * Creates a new {@link RestoreInProgressAllocationDecider} instance from
+     * given settings
+     *
+     * @param settings {@link Settings} to use
+     */
+    public RestoreInProgressAllocationDecider(Settings settings) {
+        super(settings);
+    }
+
+    @Override
+    public Decision canAllocate(final ShardRouting shardRouting, final RoutingNode node, final RoutingAllocation allocation) {
+        return canAllocate(shardRouting, allocation);
+    }
+
+    @Override
+    public Decision canAllocate(final ShardRouting shardRouting, final RoutingAllocation allocation) {
+        final RecoverySource recoverySource = shardRouting.recoverySource();
+        if (recoverySource == null || recoverySource.getType() != RecoverySource.Type.SNAPSHOT) {
+            return allocation.decision(Decision.YES, NAME, "ignored as shard is not being recovered from a snapshot");
+        }
+
+        final Snapshot snapshot = ((RecoverySource.SnapshotRecoverySource) recoverySource).snapshot();
+        final RestoreInProgress restoresInProgress = allocation.custom(RestoreInProgress.TYPE);
+
+        if (restoresInProgress != null) {
+            for (RestoreInProgress.Entry restoreInProgress : restoresInProgress.entries()) {
+                if (restoreInProgress.snapshot().equals(snapshot)) {
+                    RestoreInProgress.ShardRestoreStatus shardRestoreStatus = restoreInProgress.shards().get(shardRouting.shardId());
+                    if (shardRestoreStatus != null && shardRestoreStatus.state().completed() == false) {
+                        assert shardRestoreStatus.state() != RestoreInProgress.State.SUCCESS : "expected shard [" + shardRouting
+                            + "] to be in initializing state but got [" + shardRestoreStatus.state() + "]";
+                        return allocation.decision(Decision.YES, NAME, "shard is currently being restored");
+                    }
+                    break;
+                }
+            }
+        }
+        return allocation.decision(Decision.NO, NAME, "shard has failed to be restored from the snapshot [%s] because of [%s] - " +
+            "manually close or delete the index [%s] in order to retry to restore the snapshot again or use the reroute API to force the " +
+            "allocation of an empty primary shard", snapshot, shardRouting.unassignedInfo().getDetails(), shardRouting.getIndexName());
+    }
+
+    @Override
+    public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+        assert shardRouting.primary() : "must not call canForceAllocatePrimary on a non-primary shard " + shardRouting;
+        return canAllocate(shardRouting, node, allocation);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index a7d68a8197b..63b461afbd7 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -64,7 +64,6 @@ import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
@@ -534,7 +533,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp
                 RecoverySource recoverySource = initializingShard.recoverySource();
                 if (recoverySource.getType() == RecoverySource.Type.SNAPSHOT) {
                     Snapshot snapshot = ((SnapshotRecoverySource) recoverySource).snapshot();
-                    changes(snapshot).startedShards.put(initializingShard.shardId(),
+                    changes(snapshot).shards.put(initializingShard.shardId(),
                         new ShardRestoreStatus(initializingShard.currentNodeId(), RestoreInProgress.State.SUCCESS));
                 }
             }
@@ -550,7 +549,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp
                     // to restore this shard on another node if the snapshot files are corrupt. In case where a node just left or crashed,
                     // however, we only want to acknowledge the restore operation once it has been successfully restored on another node.
                     if (unassignedInfo.getFailure() != null && Lucene.isCorruptionException(unassignedInfo.getFailure().getCause())) {
-                        changes(snapshot).failedShards.put(failedShard.shardId(), new ShardRestoreStatus(failedShard.currentNodeId(),
+                        changes(snapshot).shards.put(failedShard.shardId(), new ShardRestoreStatus(failedShard.currentNodeId(),
                             RestoreInProgress.State.FAILURE, unassignedInfo.getFailure().getCause().getMessage()));
                     }
                 }
@@ -563,11 +562,24 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp
             if (unassignedShard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT &&
                 initializedShard.recoverySource().getType() != RecoverySource.Type.SNAPSHOT) {
                 Snapshot snapshot = ((SnapshotRecoverySource) unassignedShard.recoverySource()).snapshot();
-                changes(snapshot).failedShards.put(unassignedShard.shardId(), new ShardRestoreStatus(null,
+                changes(snapshot).shards.put(unassignedShard.shardId(), new ShardRestoreStatus(null,
                     RestoreInProgress.State.FAILURE, "recovery source type changed from snapshot to " + initializedShard.recoverySource()));
             }
         }
 
+        @Override
+        public void unassignedInfoUpdated(ShardRouting unassignedShard, UnassignedInfo newUnassignedInfo) {
+            RecoverySource recoverySource = unassignedShard.recoverySource();
+            if (recoverySource.getType() == RecoverySource.Type.SNAPSHOT) {
+                if (newUnassignedInfo.getLastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO) {
+                    Snapshot snapshot = ((SnapshotRecoverySource) recoverySource).snapshot();
+                    String reason = "shard could not be allocated to any of the nodes";
+                    changes(snapshot).shards.put(unassignedShard.shardId(),
+                        new ShardRestoreStatus(unassignedShard.currentNodeId(), RestoreInProgress.State.FAILURE, reason));
+                }
+            }
+        }
+
         /**
          * Helper method that creates update entry for the given shard id if such an entry does not exist yet.
          */
@@ -576,25 +588,21 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp
         }
 
         private static class Updates {
-            private Map failedShards = new HashMap<>();
-            private Map startedShards = new HashMap<>();
+            private Map shards = new HashMap<>();
         }
 
-        public RestoreInProgress applyChanges(RestoreInProgress oldRestore) {
+        public RestoreInProgress applyChanges(final RestoreInProgress oldRestore) {
             if (shardChanges.isEmpty() == false) {
                 final List entries = new ArrayList<>();
                 for (RestoreInProgress.Entry entry : oldRestore.entries()) {
                     Snapshot snapshot = entry.snapshot();
                     Updates updates = shardChanges.get(snapshot);
-                    assert Sets.haveEmptyIntersection(updates.startedShards.keySet(), updates.failedShards.keySet());
-                    if (updates.startedShards.isEmpty() == false || updates.failedShards.isEmpty() == false) {
+                    if (updates.shards.isEmpty() == false) {
                         ImmutableOpenMap.Builder shardsBuilder = ImmutableOpenMap.builder(entry.shards());
-                        for (Map.Entry startedShardEntry : updates.startedShards.entrySet()) {
-                            shardsBuilder.put(startedShardEntry.getKey(), startedShardEntry.getValue());
-                        }
-                        for (Map.Entry failedShardEntry : updates.failedShards.entrySet()) {
-                            shardsBuilder.put(failedShardEntry.getKey(), failedShardEntry.getValue());
+                        for (Map.Entry shard : updates.shards.entrySet()) {
+                            shardsBuilder.put(shard.getKey(), shard.getValue());
                         }
+
                         ImmutableOpenMap shards = shardsBuilder.build();
                         RestoreInProgress.State newState = overallState(RestoreInProgress.State.STARTED, shards);
                         entries.add(new RestoreInProgress.Entry(entry.snapshot(), newState, entry.indices(), shards));
@@ -607,7 +615,6 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp
                 return oldRestore;
             }
         }
-
     }
 
     public static RestoreInProgress.Entry restoreInProgress(ClusterState state, Snapshot snapshot) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
index 6fd3d66c8f8..176616690f0 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocatio
 import org.elasticsearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ResizeAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.RestoreInProgressAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
@@ -183,6 +184,7 @@ public class ClusterModuleTests extends ModuleTestCase {
             EnableAllocationDecider.class,
             NodeVersionAllocationDecider.class,
             SnapshotInProgressAllocationDecider.class,
+            RestoreInProgressAllocationDecider.class,
             FilterAllocationDecider.class,
             SameShardAllocationDecider.class,
             DiskThresholdDecider.class,
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
index 5cafe410d56..8be4c858655 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
@@ -25,6 +25,7 @@ import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -38,6 +39,7 @@ import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
@@ -46,7 +48,10 @@ import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
 
 import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
@@ -309,6 +314,8 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
         DiscoveryNode node1 = newNode("node1");
         MetaData.Builder metaDataBuilder = new MetaData.Builder(metaData);
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        Snapshot snapshot = new Snapshot("repo", new SnapshotId("snap", "randomId"));
+        Set snapshotIndices = new HashSet<>();
         for (ObjectCursor cursor: metaData.indices().values()) {
             Index index = cursor.value.getIndex();
             IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(cursor.value);
@@ -329,14 +336,14 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
                     routingTableBuilder.addAsFromDangling(indexMetaData);
                     break;
                 case 3:
+                    snapshotIndices.add(index.getName());
                     routingTableBuilder.addAsNewRestore(indexMetaData,
-                        new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
-                            indexMetaData.getIndex().getName()), new IntHashSet());
+                        new SnapshotRecoverySource(snapshot, Version.CURRENT, indexMetaData.getIndex().getName()), new IntHashSet());
                     break;
                 case 4:
+                    snapshotIndices.add(index.getName());
                     routingTableBuilder.addAsRestore(indexMetaData,
-                        new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
-                            indexMetaData.getIndex().getName()));
+                        new SnapshotRecoverySource(snapshot, Version.CURRENT, indexMetaData.getIndex().getName()));
                     break;
                 case 5:
                     routingTableBuilder.addAsNew(indexMetaData);
@@ -345,10 +352,31 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
                     throw new IndexOutOfBoundsException();
             }
         }
+
+        final RoutingTable routingTable = routingTableBuilder.build();
+
+        final ImmutableOpenMap.Builder restores = ImmutableOpenMap.builder();
+        if (snapshotIndices.isEmpty() == false) {
+            // Some indices are restored from snapshot, the RestoreInProgress must be set accordingly
+            ImmutableOpenMap.Builder restoreShards = ImmutableOpenMap.builder();
+            for (ShardRouting shard : routingTable.allShards()) {
+                if (shard.primary() && shard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT) {
+                    ShardId shardId = shard.shardId();
+                    restoreShards.put(shardId, new RestoreInProgress.ShardRestoreStatus(node1.getId(), RestoreInProgress.State.INIT));
+                }
+            }
+
+            RestoreInProgress.Entry restore = new RestoreInProgress.Entry(snapshot, RestoreInProgress.State.INIT,
+                new ArrayList<>(snapshotIndices), restoreShards.build());
+            restores.put(RestoreInProgress.TYPE, new RestoreInProgress(restore));
+        }
+
         return ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .nodes(DiscoveryNodes.builder().add(node1))
             .metaData(metaDataBuilder.build())
-            .routingTable(routingTableBuilder.build()).build();
+            .routingTable(routingTable)
+            .customs(restores.build())
+            .build();
     }
 
     private void addInSyncAllocationIds(Index index, IndexMetaData.Builder indexMetaData,
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
new file mode 100644
index 00000000000..49d69272af6
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.routing.allocation.decider;
+
+import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.RestoreInProgress;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static java.util.Collections.singletonList;
+
+/**
+ * Test {@link RestoreInProgressAllocationDecider}
+ */
+public class RestoreInProgressAllocationDeciderTests extends ESAllocationTestCase {
+
+    public void testCanAllocatePrimary() {
+        ClusterState clusterState = createInitialClusterState();
+        ShardRouting shard;
+        if (randomBoolean()) {
+            shard = clusterState.getRoutingTable().shardRoutingTable("test", 0).primaryShard();
+            assertEquals(RecoverySource.Type.EMPTY_STORE, shard.recoverySource().getType());
+        }  else {
+            shard = clusterState.getRoutingTable().shardRoutingTable("test", 0).replicaShards().get(0);
+            assertEquals(RecoverySource.Type.PEER, shard.recoverySource().getType());
+        }
+
+        final Decision decision = executeAllocation(clusterState, shard);
+        assertEquals(Decision.Type.YES, decision.type());
+        assertEquals("ignored as shard is not being recovered from a snapshot", decision.getExplanation());
+    }
+
+    public void testCannotAllocatePrimaryMissingInRestoreInProgress() {
+        ClusterState clusterState = createInitialClusterState();
+        RoutingTable routingTable = RoutingTable.builder(clusterState.getRoutingTable())
+            .addAsRestore(clusterState.getMetaData().index("test"), createSnapshotRecoverySource("_missing"))
+            .build();
+
+        clusterState = ClusterState.builder(clusterState)
+            .routingTable(routingTable)
+            .build();
+
+        ShardRouting primary = clusterState.getRoutingTable().shardRoutingTable("test", 0).primaryShard();
+        assertEquals(ShardRoutingState.UNASSIGNED, primary.state());
+        assertEquals(RecoverySource.Type.SNAPSHOT, primary.recoverySource().getType());
+
+        final Decision decision = executeAllocation(clusterState, primary);
+        assertEquals(Decision.Type.NO, decision.type());
+        assertEquals("shard has failed to be restored from the snapshot [_repository:_missing/_uuid] because of " +
+            "[restore_source[_repository/_missing]] - manually close or delete the index [test] in order to retry to restore " +
+            "the snapshot again or use the reroute API to force the allocation of an empty primary shard", decision.getExplanation());
+    }
+
+    public void testCanAllocatePrimaryExistingInRestoreInProgress() {
+        RecoverySource.SnapshotRecoverySource recoverySource = createSnapshotRecoverySource("_existing");
+
+        ClusterState clusterState = createInitialClusterState();
+        RoutingTable routingTable = RoutingTable.builder(clusterState.getRoutingTable())
+            .addAsRestore(clusterState.getMetaData().index("test"), recoverySource)
+            .build();
+
+        clusterState = ClusterState.builder(clusterState)
+            .routingTable(routingTable)
+            .build();
+
+        ShardRouting primary = clusterState.getRoutingTable().shardRoutingTable("test", 0).primaryShard();
+        assertEquals(ShardRoutingState.UNASSIGNED, primary.state());
+        assertEquals(RecoverySource.Type.SNAPSHOT, primary.recoverySource().getType());
+
+        routingTable = clusterState.routingTable();
+
+        final RestoreInProgress.State shardState;
+        if (randomBoolean()) {
+            shardState = randomFrom(RestoreInProgress.State.STARTED, RestoreInProgress.State.INIT);
+        } else {
+            shardState = RestoreInProgress.State.FAILURE;
+
+            UnassignedInfo currentInfo = primary.unassignedInfo();
+            UnassignedInfo newInfo = new UnassignedInfo(currentInfo.getReason(), currentInfo.getMessage(), new IOException("i/o failure"),
+                currentInfo.getNumFailedAllocations(), currentInfo.getUnassignedTimeInNanos(),
+                currentInfo.getUnassignedTimeInMillis(), currentInfo.isDelayed(), currentInfo.getLastAllocationStatus());
+            primary = primary.updateUnassigned(newInfo, primary.recoverySource());
+
+            IndexRoutingTable indexRoutingTable = routingTable.index("test");
+            IndexRoutingTable.Builder newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
+            for (final ObjectCursor shardEntry : indexRoutingTable.getShards().values()) {
+                final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
+                for (ShardRouting shardRouting : shardRoutingTable.getShards()) {
+                    if (shardRouting.primary()) {
+                        newIndexRoutingTable.addShard(primary);
+                    } else {
+                        newIndexRoutingTable.addShard(shardRouting);
+                    }
+                }
+            }
+            routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
+        }
+
+        ImmutableOpenMap.Builder shards = ImmutableOpenMap.builder();
+        shards.put(primary.shardId(), new RestoreInProgress.ShardRestoreStatus(clusterState.getNodes().getLocalNodeId(), shardState));
+
+        Snapshot snapshot = recoverySource.snapshot();
+        RestoreInProgress.State restoreState = RestoreInProgress.State.STARTED;
+        RestoreInProgress.Entry restore = new RestoreInProgress.Entry(snapshot, restoreState, singletonList("test"), shards.build());
+
+        clusterState = ClusterState.builder(clusterState)
+            .putCustom(RestoreInProgress.TYPE, new RestoreInProgress(restore))
+            .routingTable(routingTable)
+            .build();
+
+        Decision decision = executeAllocation(clusterState, primary);
+        if (shardState == RestoreInProgress.State.FAILURE) {
+            assertEquals(Decision.Type.NO, decision.type());
+            assertEquals("shard has failed to be restored from the snapshot [_repository:_existing/_uuid] because of " +
+                "[restore_source[_repository/_existing], failure IOException[i/o failure]] - manually close or delete the index " +
+                "[test] in order to retry to restore the snapshot again or use the reroute API to force the allocation of " +
+                "an empty primary shard", decision.getExplanation());
+        } else {
+            assertEquals(Decision.Type.YES, decision.type());
+            assertEquals("shard is currently being restored", decision.getExplanation());
+        }
+    }
+
+    private ClusterState createInitialClusterState() {
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
+            .build();
+
+        RoutingTable routingTable = RoutingTable.builder()
+            .addAsNew(metaData.index("test"))
+            .build();
+
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
+            .add(newNode("master", Collections.singleton(DiscoveryNode.Role.MASTER)))
+            .localNodeId("master")
+            .masterNodeId("master")
+            .build();
+
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .routingTable(routingTable)
+            .nodes(discoveryNodes)
+            .build();
+
+        assertEquals(2, clusterState.getRoutingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size());
+        return clusterState;
+    }
+
+    private Decision executeAllocation(final ClusterState clusterState, final ShardRouting shardRouting) {
+        final AllocationDecider decider = new RestoreInProgressAllocationDecider(Settings.EMPTY);
+        final RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.singleton(decider)),
+            clusterState.getRoutingNodes(), clusterState, null, 0L);
+        allocation.debugDecision(true);
+
+        final Decision decision;
+        if (randomBoolean()) {
+            decision = decider.canAllocate(shardRouting, allocation);
+        } else {
+            DiscoveryNode node = clusterState.getNodes().getMasterNode();
+            decision = decider.canAllocate(shardRouting, new RoutingNode(node.getId(), node), allocation);
+        }
+        return decision;
+    }
+
+    private RecoverySource.SnapshotRecoverySource createSnapshotRecoverySource(final String snapshotName) {
+        Snapshot snapshot = new Snapshot("_repository", new SnapshotId(snapshotName, "_uuid"));
+        return new RecoverySource.SnapshotRecoverySource(snapshot, Version.CURRENT, "test");
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index fa3920c1e8d..3f9c80f3ffa 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -46,6 +46,7 @@ import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.SnapshotsInProgress.Entry;
 import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus;
@@ -55,6 +56,10 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
@@ -97,14 +102,15 @@ import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING;
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAliasesExist;
@@ -117,9 +123,11 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertInde
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.not;
@@ -824,6 +832,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         prepareCreate("test-idx").setSettings(Settings.builder().put("index.allocation.max_retries", Integer.MAX_VALUE)).get();
         ensureGreen();
 
+        final NumShards numShards = getNumShards("test-idx");
+
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
             index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
@@ -848,14 +858,31 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         logger.info("--> delete index");
         cluster().wipeIndices("test-idx");
         logger.info("--> restore index after deletion");
-        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
-        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-        SearchResponse countResponse = client.prepareSearch("test-idx").setSize(0).get();
-        assertThat(countResponse.getHits().getTotalHits(), equalTo(100L));
+        final RestoreSnapshotResponse restoreResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
+                                                                                .setWaitForCompletion(true)
+                                                                                .get();
+
         logger.info("--> total number of simulated failures during restore: [{}]", getFailureCount("test-repo"));
+        final RestoreInfo restoreInfo = restoreResponse.getRestoreInfo();
+        assertThat(restoreInfo.totalShards(), equalTo(numShards.numPrimaries));
+
+        if (restoreInfo.successfulShards() == restoreInfo.totalShards()) {
+            // All shards were restored, we must find the exact number of hits
+            assertHitCount(client.prepareSearch("test-idx").setSize(0).get(), 100L);
+        } else {
+            // One or more shards failed to be restored. This can happen when there is
+            // only 1 data node: a shard failed because of the random IO exceptions
+            // during restore and then we don't allow the shard to be assigned on the
+            // same node again during the same reroute operation. Then another reroute
+            // operation is scheduled, but the RestoreInProgressAllocationDecider will
+            // block the shard to be assigned again because it failed during restore.
+            final ClusterStateResponse clusterStateResponse = client.admin().cluster().prepareState().get();
+            assertEquals(1, clusterStateResponse.getState().getNodes().getDataNodes().size());
+            assertEquals(restoreInfo.failedShards(),
+                clusterStateResponse.getState().getRoutingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size());
+        }
     }
 
-    @TestLogging("org.elasticsearch.cluster.routing:TRACE,org.elasticsearch.snapshots:TRACE")
     public void testDataFileCorruptionDuringRestore() throws Exception {
         Path repositoryLocation = randomRepoPath();
         Client client = client();
@@ -907,6 +934,155 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         cluster().wipeIndices("test-idx");
     }
 
+    /**
+     * Test that restoring a snapshot whose files can't be downloaded at all is not stuck or
+     * does not hang indefinitely.
+     */
+    public void testUnrestorableFilesDuringRestore() throws Exception {
+        final String indexName = "unrestorable-files";
+        final int maxRetries = randomIntBetween(1, 10);
+
+        Settings createIndexSettings = Settings.builder().put(SETTING_ALLOCATION_MAX_RETRY.getKey(), maxRetries).build();
+
+        Settings repositorySettings = Settings.builder()
+                                                .put("random", randomAlphaOfLength(10))
+                                                .put("max_failure_number", 10000000L)
+                                                // No lucene corruptions, we want to test retries
+                                                .put("use_lucene_corruption", false)
+                                                // Restoring a file will never complete
+                                                .put("random_data_file_io_exception_rate", 1.0)
+                                                .build();
+
+        Consumer checkUnassignedInfo = unassignedInfo -> {
+            assertThat(unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
+            assertThat(unassignedInfo.getNumFailedAllocations(), anyOf(equalTo(maxRetries), equalTo(1)));
+        };
+
+        unrestorableUseCase(indexName, createIndexSettings, repositorySettings, Settings.EMPTY, checkUnassignedInfo, () -> {});
+    }
+
+    /**
+     * Test that restoring an index with shard allocation filtering settings that prevents
+     * its allocation does not hang indefinitely.
+     */
+    public void testUnrestorableIndexDuringRestore() throws Exception {
+        final String indexName = "unrestorable-index";
+        Settings restoreIndexSettings = Settings.builder().put("index.routing.allocation.include._name", randomAlphaOfLength(5)).build();
+
+        Consumer checkUnassignedInfo = unassignedInfo -> {
+            assertThat(unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.NEW_INDEX_RESTORED));
+        };
+
+        Runnable fixupAction =() -> {
+            // remove the shard allocation filtering settings and use the Reroute API to retry the failed shards
+            assertAcked(client().admin().indices().prepareUpdateSettings(indexName)
+                                                    .setSettings(Settings.builder()
+                                                                            .putNull("index.routing.allocation.include._name")
+                                                                            .build()));
+            assertAcked(client().admin().cluster().prepareReroute().setRetryFailed(true));
+        };
+
+        unrestorableUseCase(indexName, Settings.EMPTY, Settings.EMPTY, restoreIndexSettings, checkUnassignedInfo, fixupAction);
+    }
+
+    /** Execute the unrestorable test use case **/
+    private void unrestorableUseCase(final String indexName,
+                                     final Settings createIndexSettings,
+                                     final Settings repositorySettings,
+                                     final Settings restoreIndexSettings,
+                                     final Consumer checkUnassignedInfo,
+                                     final Runnable fixUpAction) throws Exception {
+        // create a test repository
+        final Path repositoryLocation = randomRepoPath();
+        assertAcked(client().admin().cluster().preparePutRepository("test-repo")
+                                                .setType("fs")
+                                                .setSettings(Settings.builder().put("location", repositoryLocation)));
+        // create a test index
+        assertAcked(prepareCreate(indexName, Settings.builder().put(createIndexSettings)));
+
+        // index some documents
+        final int nbDocs = scaledRandomIntBetween(10, 100);
+        for (int i = 0; i < nbDocs; i++) {
+            index(indexName, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        flushAndRefresh(indexName);
+        assertThat(client().prepareSearch(indexName).setSize(0).get().getHits().getTotalHits(), equalTo((long) nbDocs));
+
+        // create a snapshot
+        final NumShards numShards = getNumShards(indexName);
+        CreateSnapshotResponse snapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
+                                                                            .setWaitForCompletion(true)
+                                                                            .setIndices(indexName)
+                                                                            .get();
+
+        assertThat(snapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(snapshotResponse.getSnapshotInfo().successfulShards(), equalTo(numShards.numPrimaries));
+        assertThat(snapshotResponse.getSnapshotInfo().failedShards(), equalTo(0));
+
+        // delete the test index
+        assertAcked(client().admin().indices().prepareDelete(indexName));
+
+        // update the test repository
+        assertAcked(client().admin().cluster().preparePutRepository("test-repo")
+                                              .setType("mock")
+                                              .setSettings(Settings.builder()
+                                                                   .put("location", repositoryLocation)
+                                                                   .put(repositorySettings)
+                                                                   .build()));
+
+        // attempt to restore the snapshot with the given settings
+        RestoreSnapshotResponse restoreResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
+                                                                            .setIndices(indexName)
+                                                                            .setIndexSettings(restoreIndexSettings)
+                                                                            .setWaitForCompletion(true)
+                                                                            .get();
+
+        // check that all shards failed during restore
+        assertThat(restoreResponse.getRestoreInfo().totalShards(), equalTo(numShards.numPrimaries));
+        assertThat(restoreResponse.getRestoreInfo().successfulShards(), equalTo(0));
+
+        ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().setCustoms(true).setRoutingTable(true).get();
+
+        // check that there is no restore in progress
+        RestoreInProgress restoreInProgress = clusterStateResponse.getState().custom(RestoreInProgress.TYPE);
+        assertNotNull("RestoreInProgress must be not null", restoreInProgress);
+        assertThat("RestoreInProgress must be empty", restoreInProgress.entries(), hasSize(0));
+
+        // check that the shards have been created but are not assigned
+        assertThat(clusterStateResponse.getState().getRoutingTable().allShards(indexName), hasSize(numShards.totalNumShards));
+
+        // check that every primary shard is unassigned
+        for (ShardRouting shard : clusterStateResponse.getState().getRoutingTable().allShards(indexName)) {
+            if (shard.primary()) {
+                assertThat(shard.state(), equalTo(ShardRoutingState.UNASSIGNED));
+                assertThat(shard.recoverySource().getType(), equalTo(RecoverySource.Type.SNAPSHOT));
+                assertThat(shard.unassignedInfo().getLastAllocationStatus(), equalTo(UnassignedInfo.AllocationStatus.DECIDERS_NO));
+                checkUnassignedInfo.accept(shard.unassignedInfo());
+            }
+        }
+
+        // update the test repository in order to make it work
+        assertAcked(client().admin().cluster().preparePutRepository("test-repo")
+                                              .setType("fs")
+                                              .setSettings(Settings.builder().put("location", repositoryLocation)));
+
+        // execute action to eventually fix the situation
+        fixUpAction.run();
+
+        // delete the index and restore again
+        assertAcked(client().admin().indices().prepareDelete(indexName));
+
+        restoreResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).get();
+        assertThat(restoreResponse.getRestoreInfo().totalShards(), equalTo(numShards.numPrimaries));
+        assertThat(restoreResponse.getRestoreInfo().successfulShards(), equalTo(numShards.numPrimaries));
+
+        // Wait for the shards to be assigned
+        ensureGreen(indexName);
+        refresh(indexName);
+
+        assertThat(client().prepareSearch(indexName).setSize(0).get().getHits().getTotalHits(), equalTo((long) nbDocs));
+    }
+
     public void testDeletionOfFailingToRecoverIndexShouldStopRestore() throws Exception {
         Path repositoryLocation = randomRepoPath();
         Client client = client();

From 77617c8e62f7fa1407c62bdf33e4892206d62952 Mon Sep 17 00:00:00 2001
From: Alan Woodward 
Date: Tue, 12 Dec 2017 13:33:37 +0000
Subject: [PATCH 246/297] [TEST] Add test for *_range fields in query_string
 queries (#27756)

[TEST] Add test for *_range fields in query_string queries

Closes #26555
---
 ...angeFieldQueryStringQueryBuilderTests.java | 145 ++++++++++++++++++
 .../rest-api-spec/test/range/10_basic.yml     |  36 +++++
 2 files changed, 181 insertions(+)
 create mode 100644 modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java

diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java
new file mode 100644
index 00000000000..258084bdab9
--- /dev/null
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.document.DoubleRange;
+import org.apache.lucene.document.FloatRange;
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.document.InetAddressRange;
+import org.apache.lucene.document.IntRange;
+import org.apache.lucene.document.LongRange;
+import org.apache.lucene.queries.BinaryDocValuesRangeQuery;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.PointRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.joda.DateMathParser;
+import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryStringQueryBuilder;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.AbstractQueryTestCase;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.either;
+import static org.hamcrest.core.IsInstanceOf.instanceOf;
+
+public class RangeFieldQueryStringQueryBuilderTests extends AbstractQueryTestCase {
+
+    private static final String INTEGER_RANGE_FIELD_NAME = "mapped_int_range";
+    private static final String LONG_RANGE_FIELD_NAME = "mapped_long_range";
+    private static final String FLOAT_RANGE_FIELD_NAME = "mapped_float_range";
+    private static final String DOUBLE_RANGE_FIELD_NAME = "mapped_double_range";
+    private static final String DATE_RANGE_FIELD_NAME = "mapped_date_range";
+    private static final String IP_RANGE_FIELD_NAME = "mapped_ip_range";
+
+    @Override
+    protected Collection> getPlugins() {
+        return Collections.singleton(MapperExtrasPlugin.class);
+    }
+
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+        mapperService.merge("doc", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("doc",
+            INTEGER_RANGE_FIELD_NAME, "type=integer_range",
+            LONG_RANGE_FIELD_NAME, "type=long_range",
+            FLOAT_RANGE_FIELD_NAME, "type=float_range",
+            DOUBLE_RANGE_FIELD_NAME, "type=double_range",
+            DATE_RANGE_FIELD_NAME, "type=date_range",
+            IP_RANGE_FIELD_NAME, "type=ip_range"
+        ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+
+    }
+
+    public void testIntegerRangeQuery() throws Exception {
+        Query query = new QueryStringQueryBuilder(INTEGER_RANGE_FIELD_NAME + ":[-450 TO 45000]").toQuery(createShardContext());
+        Query range = IntRange.newIntersectsQuery(INTEGER_RANGE_FIELD_NAME, new int[]{-450}, new int[]{45000});
+        Query dv = RangeFieldMapper.RangeType.INTEGER.dvRangeQuery(INTEGER_RANGE_FIELD_NAME,
+            BinaryDocValuesRangeQuery.QueryType.INTERSECTS, -450, 45000, true, true);
+        assertEquals(new IndexOrDocValuesQuery(range, dv), query);
+    }
+
+    public void testLongRangeQuery() throws Exception {
+        Query query = new QueryStringQueryBuilder(LONG_RANGE_FIELD_NAME + ":[-450 TO 45000]").toQuery(createShardContext());
+        Query range = LongRange.newIntersectsQuery(LONG_RANGE_FIELD_NAME, new long[]{-450}, new long[]{45000});
+        Query dv = RangeFieldMapper.RangeType.LONG.dvRangeQuery(LONG_RANGE_FIELD_NAME,
+            BinaryDocValuesRangeQuery.QueryType.INTERSECTS, -450, 45000, true, true);
+        assertEquals(new IndexOrDocValuesQuery(range, dv), query);
+    }
+
+    public void testFloatRangeQuery() throws Exception {
+        Query query = new QueryStringQueryBuilder(FLOAT_RANGE_FIELD_NAME + ":[-450 TO 45000]").toQuery(createShardContext());
+        Query range = FloatRange.newIntersectsQuery(FLOAT_RANGE_FIELD_NAME, new float[]{-450}, new float[]{45000});
+        Query dv = RangeFieldMapper.RangeType.FLOAT.dvRangeQuery(FLOAT_RANGE_FIELD_NAME,
+            BinaryDocValuesRangeQuery.QueryType.INTERSECTS, -450.0f, 45000.0f, true, true);
+        assertEquals(new IndexOrDocValuesQuery(range, dv), query);
+    }
+
+    public void testDoubleRangeQuery() throws Exception {
+        Query query = new QueryStringQueryBuilder(DOUBLE_RANGE_FIELD_NAME + ":[-450 TO 45000]").toQuery(createShardContext());
+        Query range = DoubleRange.newIntersectsQuery(DOUBLE_RANGE_FIELD_NAME, new double[]{-450}, new double[]{45000});
+        Query dv = RangeFieldMapper.RangeType.DOUBLE.dvRangeQuery(DOUBLE_RANGE_FIELD_NAME,
+            BinaryDocValuesRangeQuery.QueryType.INTERSECTS, -450.0, 45000.0, true, true);
+        assertEquals(new IndexOrDocValuesQuery(range, dv), query);
+    }
+
+    public void testDateRangeQuery() throws Exception {
+        QueryShardContext context = createShardContext();
+        RangeFieldMapper.RangeFieldType type = (RangeFieldMapper.RangeFieldType) context.fieldMapper(DATE_RANGE_FIELD_NAME);
+        DateMathParser parser = type.dateMathParser;
+        Query query = new QueryStringQueryBuilder(DATE_RANGE_FIELD_NAME + ":[2010-01-01 TO 2018-01-01]").toQuery(createShardContext());
+        Query range = LongRange.newIntersectsQuery(DATE_RANGE_FIELD_NAME,
+            new long[]{ parser.parse("2010-01-01", () -> 0)}, new long[]{ parser.parse("2018-01-01", () -> 0)});
+        Query dv = RangeFieldMapper.RangeType.DATE.dvRangeQuery(DATE_RANGE_FIELD_NAME,
+            BinaryDocValuesRangeQuery.QueryType.INTERSECTS,
+            parser.parse("2010-01-01", () -> 0),
+            parser.parse("2018-01-01", () -> 0), true, true);
+        assertEquals(new IndexOrDocValuesQuery(range, dv), query);
+    }
+
+    public void testIPRangeQuery() throws Exception {
+        InetAddress lower = InetAddresses.forString("192.168.0.1");
+        InetAddress upper = InetAddresses.forString("192.168.0.5");
+        Query query = new QueryStringQueryBuilder(IP_RANGE_FIELD_NAME + ":[192.168.0.1 TO 192.168.0.5]").toQuery(createShardContext());
+        Query range = InetAddressRange.newIntersectsQuery(IP_RANGE_FIELD_NAME, lower, upper);
+        Query dv = RangeFieldMapper.RangeType.IP.dvRangeQuery(IP_RANGE_FIELD_NAME,
+            BinaryDocValuesRangeQuery.QueryType.INTERSECTS,
+            lower, upper, true, true);
+        assertEquals(new IndexOrDocValuesQuery(range, dv), query);
+    }
+
+    @Override
+    protected QueryStringQueryBuilder doCreateTestQueryBuilder() {
+        return new QueryStringQueryBuilder(INTEGER_RANGE_FIELD_NAME + ":[-450 TO 450]");
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(QueryStringQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
+        assertThat(query, either(instanceOf(PointRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class)));
+    }
+}
diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/range/10_basic.yml b/modules/mapper-extras/src/test/resources/rest-api-spec/test/range/10_basic.yml
index 9fd54d6342d..4bf6e3e2b4a 100644
--- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/range/10_basic.yml
+++ b/modules/mapper-extras/src/test/resources/rest-api-spec/test/range/10_basic.yml
@@ -55,6 +55,12 @@ setup:
 
   - match: { hits.total: 3 }
 
+  - do:
+      search:
+        body: { "size" : 0, "query" : { "query_string" : { "query" : "integer_range:[3 TO 4]" } } }
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         body: { "size" : 0, "query" : { "range" : { "integer_range" : { "gte": 3, "lte" : 4, "relation": "intersects" } } } }
@@ -107,6 +113,12 @@ setup:
 
   - match: { hits.total: 3 }
 
+  - do:
+      search:
+        body: { "size" : 0, "query" : { "query_string" : { "query" : "long_range:[3 TO 4]" } } }
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         body: { "size" : 0, "query" : { "range" : { "long_range" : { "gte": 3, "lte" : 4, "relation": "intersects" } } } }
@@ -159,6 +171,12 @@ setup:
 
   - match: { hits.total: 3 }
 
+  - do:
+      search:
+        body: { "size" : 0, "query" : { "query_string" : { "query" : "float_range:[3 TO 4]" } } }
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         body: { "size" : 0, "query" : { "range" : { "float_range" : { "gte": 3, "lte" : 4, "relation": "intersects" } } } }
@@ -211,6 +229,12 @@ setup:
 
   - match: { hits.total: 3 }
 
+  - do:
+      search:
+        body: { "size" : 0, "query" : { "query_string" : { "query" : "double_range:[3 TO 4]" } } }
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         body: { "size" : 0, "query" : { "range" : { "double_range" : { "gte": 3, "lte" : 4, "relation": "intersects" } } } }
@@ -263,6 +287,12 @@ setup:
 
   - match: { hits.total: 3 }
 
+  - do:
+      search:
+        body: { "size" : 0, "query" : { "query_string" : { "query" : "ip_range:[192.168.0.3 TO 192.168.0.4]" } } }
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         body: { "size" : 0, "query" : { "range" : { "ip_range" : { "gte": "192.168.0.3", "lte" : "192.168.0.4", "relation": "intersects" } } } }
@@ -315,6 +345,12 @@ setup:
 
   - match: { hits.total: 3 }
 
+  - do:
+      search:
+        body: { "size" : 0, "query" : { "query_string" : { "query" : "date_range:[2017-09-03 TO 2017-09-04]" } } }
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         body: { "size" : 0, "query" : { "range" : { "date_range" : { "gte": "2017-09-03", "lte" : "2017-09-04", "relation": "intersects" } } } }

From 008296e2b6f58dcd5c3d339447f024c720f14ca6 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Tue, 12 Dec 2017 10:24:37 -0500
Subject: [PATCH 247/297] Reorganize configuring Elasticsearch docs

This commit reorganizes some of the content in the configuring
Elasticsearch section of the docs. The changes are:
 - move JVM options out of system configuration into configuring
   Elasticsearch
 - move JVM options to its own page of the docs
 - move configuring the heap to important Elasticsearch settings
 - move configuring the heap to its own page of the docs
 - move all important settings to individual pages in the docs
 - remove bootstrap.memory_lock from important settings, this is covered
   in the swap section of system configuration

Relates #27755
---
 .../tools/launchers/JvmOptionsParser.java     |   8 +-
 docs/reference/setup.asciidoc                 |   2 +
 .../reference/setup/bootstrap-checks.asciidoc |   7 +-
 .../setup/important-settings.asciidoc         | 216 ++----------------
 .../important-settings/cluster-name.asciidoc  |  14 ++
 .../discovery-settings.asciidoc               |  58 +++++
 .../important-settings/gc-logging.asciidoc    |   7 +
 .../heap-dump-path.asciidoc                   |  19 ++
 .../important-settings/heap-size.asciidoc     |  72 ++++++
 .../important-settings/network-host.asciidoc  |  29 +++
 .../important-settings/node-name.asciidoc     |  22 ++
 .../important-settings/path-settings.asciidoc |  32 +++
 docs/reference/setup/jvm-options.asciidoc     |  82 +++++++
 docs/reference/setup/sysconfig.asciidoc       |   6 +-
 .../setup/sysconfig/configuring.asciidoc      |  57 -----
 .../setup/sysconfig/heap_size.asciidoc        |  74 ------
 docs/reference/setup/sysconfig/swap.asciidoc  |   2 +-
 17 files changed, 365 insertions(+), 342 deletions(-)
 create mode 100644 docs/reference/setup/important-settings/cluster-name.asciidoc
 create mode 100644 docs/reference/setup/important-settings/discovery-settings.asciidoc
 create mode 100644 docs/reference/setup/important-settings/gc-logging.asciidoc
 create mode 100644 docs/reference/setup/important-settings/heap-dump-path.asciidoc
 create mode 100644 docs/reference/setup/important-settings/heap-size.asciidoc
 create mode 100644 docs/reference/setup/important-settings/network-host.asciidoc
 create mode 100644 docs/reference/setup/important-settings/node-name.asciidoc
 create mode 100644 docs/reference/setup/important-settings/path-settings.asciidoc
 create mode 100644 docs/reference/setup/jvm-options.asciidoc
 delete mode 100644 docs/reference/setup/sysconfig/heap_size.asciidoc

diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java
index fe7e045e6bc..8cd401e5322 100644
--- a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java
+++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java
@@ -137,12 +137,12 @@ final class JvmOptionsParser {
      *         only
      *     
        • 
      *     
      • 
-     *         a line starting with a number followed by a dash is treated as a JVM option that applies to the matching Java specified major
-     *         version and all larger Java major versions
+     *         a line starting with a number followed by a dash followed by a colon is treated as a JVM option that applies to the matching
+     *         Java specified major version and all larger Java major versions
      *     
        • 
      *     
      • 
-     *         a line starting with a number followed by a dash followed by a number is treated as a JVM option that applies to the
-     *         specified range of matching Java major versions
+     *         a line starting with a number followed by a dash followed by a number followed by a colon is treated as a JVM option that
+     *         applies to the specified range of matching Java major versions
      *     
        • 
      * 
      
      *
diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc
index 0dcafe375cf..814abcd6527 100644
--- a/docs/reference/setup.asciidoc
+++ b/docs/reference/setup.asciidoc
@@ -40,6 +40,8 @@ include::setup/install.asciidoc[]
 
 include::setup/configuration.asciidoc[]
 
+include::setup/jvm-options.asciidoc[]
+
 include::setup/secure-settings.asciidoc[]
 
 include::setup/logging-config.asciidoc[]
diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
index fa099efcf7f..1e2b5f8b30d 100644
--- a/docs/reference/setup/bootstrap-checks.asciidoc
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -67,13 +67,12 @@ If a JVM is started with unequal initial and max heap size, it can be
 prone to pauses as the JVM heap is resized during system usage. To avoid
 these resize pauses, it's best to start the JVM with the initial heap
 size equal to the maximum heap size. Additionally, if
-<> is enabled, the JVM
+<> is enabled, the JVM
 will lock the initial size of the heap on startup. If the initial heap
 size is not equal to the maximum heap size, after a resize it will not
 be the case that all of the JVM heap is locked in memory. To pass the
 heap size check, you must configure the <>.
 
-
 === File descriptor check
 
 File descriptors are a Unix construct for tracking open "files". In Unix
@@ -95,13 +94,13 @@ Elasticsearch would much rather use to service requests. There are
 several ways to configure a system to disallow swapping. One way is by
 requesting the JVM to lock the heap in memory through `mlockall` (Unix)
 or virtual lock (Windows). This is done via the Elasticsearch setting
-<>. However, there are
+<>. However, there are
 cases where this setting can be passed to Elasticsearch but
 Elasticsearch is not able to lock the heap (e.g., if the `elasticsearch`
 user does not have `memlock unlimited`). The memory lock check verifies
 that *if* the `bootstrap.memory_lock` setting is enabled, that the JVM
 was successfully able to lock the heap. To pass the memory lock check,
-you might have to configure <>.
+you might have to configure <>.
 
 [[max-number-threads-check]]
 === Maximum number of threads check
diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc
index 5186a8670e2..997f267a7e2 100644
--- a/docs/reference/setup/important-settings.asciidoc
+++ b/docs/reference/setup/important-settings.asciidoc
@@ -2,211 +2,31 @@
 == Important Elasticsearch configuration
 
 While Elasticsearch requires very little configuration, there are a number of
-settings which need to be configured manually and should definitely be
-configured before going into production.
+settings which need to be considered before going into production.
 
-* <>
-* <>
-* <>
-* <>
-* <>
-* <>
-* <>
-* <>
+The following settings *must* be considered before going to production:
 
-[float]
-[[path-settings]]
-=== `path.data` and `path.logs`
+* <>
+* <>
+* <>
+* <>
+* <>
+* <>
+* <>
+* <>
 
-If you are using the `.zip` or `.tar.gz` archives, the `data` and `logs`
-directories are sub-folders of `$ES_HOME`.  If these important folders are
-left in their default locations, there is a high risk of them being deleted
-while  upgrading Elasticsearch to a new version.
+include::important-settings/path-settings.asciidoc[]
 
-In production use, you will almost certainly want to change the locations of
-the data and log folder:
+include::important-settings/cluster-name.asciidoc[]
 
-[source,yaml]
---------------------------------------------------
-path:
-  logs: /var/log/elasticsearch
-  data: /var/data/elasticsearch
---------------------------------------------------
+include::important-settings/node-name.asciidoc[]
 
-The RPM and Debian distributions already use custom paths for `data` and
-`logs`.
+include::important-settings/network-host.asciidoc[]
 
-The `path.data` settings can be set to multiple paths, in which case all paths
-will be used to store data (although the files belonging to a single shard
-will all be stored on the same data path):
+include::important-settings/discovery-settings.asciidoc[]
 
-[source,yaml]
---------------------------------------------------
-path:
-  data:
-    - /mnt/elasticsearch_1
-    - /mnt/elasticsearch_2
-    - /mnt/elasticsearch_3
---------------------------------------------------
+include::important-settings/heap-size.asciidoc[]
 
-[float]
-[[cluster.name]]
-=== `cluster.name`
+include::important-settings/heap-dump-path.asciidoc[]
 
-A node can only join a cluster when it shares its `cluster.name` with all the
-other nodes in the cluster. The default name is `elasticsearch`, but you
-should change it to an appropriate name which describes the purpose of the
-cluster.
-
-[source,yaml]
---------------------------------------------------
-cluster.name: logging-prod
---------------------------------------------------
-
-Make sure that you don't reuse the same cluster names in different
-environments, otherwise you might end up with nodes joining the wrong cluster.
-
-[float]
-[[node.name]]
-=== `node.name`
-
-By default, Elasticsearch will take the 7 first character of the randomly generated uuid used as the node id.
-Note that the node id is persisted and does not change when a node restarts and therefore the default node name
-will also not change.
-
-It is worth configuring a more meaningful name which will also have the
-advantage of persisting after restarting the node:
-
-[source,yaml]
---------------------------------------------------
-node.name: prod-data-2
---------------------------------------------------
-
-The `node.name` can also be set to the server's HOSTNAME as follows:
-
-[source,yaml]
---------------------------------------------------
-node.name: ${HOSTNAME}
---------------------------------------------------
-
-[float]
-[[bootstrap.memory_lock]]
-=== `bootstrap.memory_lock`
-
-It is vitally important to the health of your node that none of the JVM is
-ever swapped out to disk.  One way of achieving that is set the
-`bootstrap.memory_lock` setting to `true`.
-
-For this setting to have effect, other system settings need to be configured
-first. See <> for more details about how to set up memory locking
-correctly.
-
-[float]
-[[network.host]]
-=== `network.host`
-
-By default, Elasticsearch binds to loopback addresses only -- e.g. `127.0.0.1`
-and `[::1]`. This is sufficient to run a single development node on a server.
-
-TIP: In fact, more than one node can be started from the same `$ES_HOME` location
-on a single node.  This can be useful for testing Elasticsearch's ability to
-form clusters, but it is not a configuration recommended for production.
-
-In order to communicate and to form a cluster with nodes on other servers,
-your node will need to bind to a non-loopback address.  While there are many
-<>, usually all you need to configure is
-`network.host`:
-
-[source,yaml]
---------------------------------------------------
-network.host: 192.168.1.10
---------------------------------------------------
-
-The `network.host` setting also understands some special values such as
-`_local_`, `_site_`, `_global_` and modifiers like `:ip4` and `:ip6`, details
-of which can be found in <>.
-
-IMPORTANT: As soon you provide a custom setting for `network.host`,
-Elasticsearch assumes that you are moving from development mode to production
-mode, and upgrades a number of system startup checks from warnings to
-exceptions.  See <> for more information.
-
-[float]
-[[unicast.hosts]]
-=== `discovery.zen.ping.unicast.hosts`
-
-Out of the box, without any network configuration, Elasticsearch will bind to
-the available loopback addresses and will scan ports 9300 to 9305 to try to
-connect to other nodes running on the same server. This provides an auto-
-clustering experience without having to do any configuration.
-
-When the moment comes to form a cluster with nodes on other servers, you have
-to provide a seed list of other nodes in the cluster that are likely to be
-live and contactable.  This can be specified as follows:
-
-[source,yaml]
---------------------------------------------------
-discovery.zen.ping.unicast.hosts:
-   - 192.168.1.10:9300
-   - 192.168.1.11 <1>
-   - seeds.mydomain.com <2>
---------------------------------------------------
-<1> The port will default to `transport.profiles.default.port` and fallback to `transport.tcp.port` if not specified.
-<2> A hostname that resolves to multiple IP addresses will try all resolved addresses.
-
-[float]
-[[minimum_master_nodes]]
-=== `discovery.zen.minimum_master_nodes`
-
-To prevent data loss, it is vital to configure the
-`discovery.zen.minimum_master_nodes` setting so that each master-eligible node
-knows the _minimum number of master-eligible nodes_ that must be visible in
-order to form a cluster.
-
-Without this setting, a cluster that suffers a network failure is at risk of
-having the cluster split into two independent clusters -- a split brain --
-which will lead to data loss. A more detailed explanation is provided
-in <>.
-
-To avoid a split brain, this setting should be set to a _quorum_ of master-
-eligible nodes:
-
-    (master_eligible_nodes / 2) + 1
-
-In other words, if there are three master-eligible nodes, then minimum master
-nodes should be set to `(3 / 2) + 1` or `2`:
-
-[source,yaml]
---------------------------------------------------
-discovery.zen.minimum_master_nodes: 2
---------------------------------------------------
-
-[float]
-[[heap-dump-path]]
-=== JVM heap dump path
-
-The <> and <> package distributions default to configuring
-the JVM to dump the heap on out of memory exceptions to
-`/var/lib/elasticsearch`. If this path is not suitable for storing heap dumps,
-you should modify the entry `-XX:HeapDumpPath=/var/lib/elasticsearch` in
-<> to an alternate path.  If you specify a filename
-instead of a directory, the JVM will repeatedly use the same file; this is one
-mechanism for preventing heap dumps from accumulating in the heap dump path.
-Alternatively, you can configure a scheduled task via your OS to remove heap
-dumps that are older than a configured age.
-
-Note that the archive distributions do not configure the heap dump path by
-default. Instead, the JVM will default to dumping to the working directory for
-the Elasticsearch process. If you wish to configure a heap dump path, you should
-modify the entry `#-XX:HeapDumpPath=/heap/dump/path` in
-<> to remove the comment marker `#` and to specify an
-actual path.
-
-[float]
-[[gc-logging]]
-=== GC logging
-
-By default, Elasticsearch enables GC logs. These are configured in
-<> and default to the same default location as the
-Elasticsearch logs. The default configuration rotates the logs every 64 MB and
-can consume up to 2 GB of disk space.
+include::important-settings/gc-logging.asciidoc[]
diff --git a/docs/reference/setup/important-settings/cluster-name.asciidoc b/docs/reference/setup/important-settings/cluster-name.asciidoc
new file mode 100644
index 00000000000..02039e34de3
--- /dev/null
+++ b/docs/reference/setup/important-settings/cluster-name.asciidoc
@@ -0,0 +1,14 @@
+[[cluster.name]]
+=== `cluster.name`
+
+A node can only join a cluster when it shares its `cluster.name` with all the
+other nodes in the cluster. The default name is `elasticsearch`, but you should
+change it to an appropriate name which describes the purpose of the cluster.
+
+[source,yaml]
+--------------------------------------------------
+cluster.name: logging-prod
+--------------------------------------------------
+
+Make sure that you don't reuse the same cluster names in different environments,
+otherwise you might end up with nodes joining the wrong cluster.
diff --git a/docs/reference/setup/important-settings/discovery-settings.asciidoc b/docs/reference/setup/important-settings/discovery-settings.asciidoc
new file mode 100644
index 00000000000..e0c67ffb22d
--- /dev/null
+++ b/docs/reference/setup/important-settings/discovery-settings.asciidoc
@@ -0,0 +1,58 @@
+[[discovery-settings]]
+=== Discovery settings
+
+Elasticsearch uses a custom discovery implementation called "Zen Discovery" for
+node-to-node clustering and master election. There are two important discovery
+settings that should be configured before going to production.
+
+[float]
+[[unicast.hosts]]
+==== `discovery.zen.ping.unicast.hosts`
+
+Out of the box, without any network configuration, Elasticsearch will bind to
+the available loopback addresses and will scan ports 9300 to 9305 to try to
+connect to other nodes running on the same server. This provides an auto-
+clustering experience without having to do any configuration.
+
+When the moment comes to form a cluster with nodes on other servers, you have to
+provide a seed list of other nodes in the cluster that are likely to be live and
+contactable. This can be specified as follows:
+
+[source,yaml]
+--------------------------------------------------
+discovery.zen.ping.unicast.hosts:
+   - 192.168.1.10:9300
+   - 192.168.1.11 <1>
+   - seeds.mydomain.com <2>
+--------------------------------------------------
+<1> The port will default to `transport.profiles.default.port` and fallback to
+    `transport.tcp.port` if not specified.
+<2> A hostname that resolves to multiple IP addresses will try all resolved
+    addresses.
+
+[float]
+[[minimum_master_nodes]]
+==== `discovery.zen.minimum_master_nodes`
+
+To prevent data loss, it is vital to configure the
+`discovery.zen.minimum_master_nodes` setting so that each master-eligible node
+knows the _minimum number of master-eligible nodes_ that must be visible in
+order to form a cluster.
+
+Without this setting, a cluster that suffers a network failure is at risk of
+having the cluster split into two independent clusters -- a split brain -- which
+will lead to data loss. A more detailed explanation is provided in
+<>.
+
+To avoid a split brain, this setting should be set to a _quorum_ of
+master-eligible nodes:
+
+    (master_eligible_nodes / 2) + 1
+
+In other words, if there are three master-eligible nodes, then minimum master
+nodes should be set to `(3 / 2) + 1` or `2`:
+
+[source,yaml]
+--------------------------------------------------
+discovery.zen.minimum_master_nodes: 2
+--------------------------------------------------
diff --git a/docs/reference/setup/important-settings/gc-logging.asciidoc b/docs/reference/setup/important-settings/gc-logging.asciidoc
new file mode 100644
index 00000000000..30df94d071f
--- /dev/null
+++ b/docs/reference/setup/important-settings/gc-logging.asciidoc
@@ -0,0 +1,7 @@
+[[gc-logging]]
+=== GC logging
+
+By default, Elasticsearch enables GC logs. These are configured in
+<> and default to the same default location as the
+Elasticsearch logs. The default configuration rotates the logs every 64 MB and
+can consume up to 2 GB of disk space.
diff --git a/docs/reference/setup/important-settings/heap-dump-path.asciidoc b/docs/reference/setup/important-settings/heap-dump-path.asciidoc
new file mode 100644
index 00000000000..def7d5962fa
--- /dev/null
+++ b/docs/reference/setup/important-settings/heap-dump-path.asciidoc
@@ -0,0 +1,19 @@
+[[heap-dump-path]]
+=== JVM heap dump path
+
+The <> and <> package distributions default to configuring
+the JVM to dump the heap on out of memory exceptions to
+`/var/lib/elasticsearch`. If this path is not suitable for storing heap dumps,
+you should modify the entry `-XX:HeapDumpPath=/var/lib/elasticsearch` in
+<> to an alternate path.  If you specify a filename
+instead of a directory, the JVM will repeatedly use the same file; this is one
+mechanism for preventing heap dumps from accumulating in the heap dump path.
+Alternatively, you can configure a scheduled task via your OS to remove heap
+dumps that are older than a configured age.
+
+Note that the archive distributions do not configure the heap dump path by
+default. Instead, the JVM will default to dumping to the working directory for
+the Elasticsearch process. If you wish to configure a heap dump path, you should
+modify the entry `#-XX:HeapDumpPath=/heap/dump/path` in
+<> to remove the comment marker `#` and to specify an
+actual path.
diff --git a/docs/reference/setup/important-settings/heap-size.asciidoc b/docs/reference/setup/important-settings/heap-size.asciidoc
new file mode 100644
index 00000000000..77aa23b61df
--- /dev/null
+++ b/docs/reference/setup/important-settings/heap-size.asciidoc
@@ -0,0 +1,72 @@
+[[heap-size]]
+=== Setting the heap size
+
+By default, Elasticsearch tells the JVM to use a heap with a minimum and maximum
+size of 1 GB. When moving to production, it is important to configure heap size
+to ensure that Elasticsearch has enough heap available.
+
+Elasticsearch will assign the entire heap specified in
+<> via the `Xms` (minimum heap size) and `Xmx` (maximum
+heap size) settings.
+
+The value for these setting depends on the amount of RAM available on your
+server. Good rules of thumb are:
+
+* Set the minimum heap size (`Xms`) and maximum heap size (`Xmx`) to be equal to
+  each other.
+
+* The more heap available to Elasticsearch, the more memory it can use for
+  caching. But note that too much heap can subject you to long garbage
+  collection pauses.
+
+* Set `Xmx` to no more than 50% of your physical RAM, to ensure that there is
+  enough physical RAM left for kernel file system caches.
+
+* Don’t set `Xmx` to above the cutoff that the JVM uses for compressed object
+  pointers (compressed oops); the exact cutoff varies but is near 32 GB. You can
+  verify that you are under the limit by looking for a line in the logs like the
+  following:
++
+    heap size [1.9gb], compressed ordinary object pointers [true]
+
+* Even better, try to stay below the threshold for zero-based compressed oops;
+  the exact cutoff varies but 26 GB is safe on most systems, but can be as large
+  as 30 GB on some systems. You can verify that you are under the limit by
+  starting Elasticsearch with the JVM options `-XX:+UnlockDiagnosticVMOptions
+  -XX:+PrintCompressedOopsMode` and looking for a line like the following:
++
+--
+    heap address: 0x000000011be00000, size: 27648 MB, zero based Compressed Oops
+
+showing that zero-based compressed oops are enabled instead of
+
+    heap address: 0x0000000118400000, size: 28672 MB, Compressed Oops with base: 0x00000001183ff000
+--
+
+Here are examples of how to set the heap size via the jvm.options file:
+
+[source,txt]
+------------------
+-Xms2g <1>
+-Xmx2g <2>
+------------------
+<1> Set the minimum heap size to 2g.
+<2> Set the maximum heap size to 2g.
+
+It is also possible to set the heap size via an environment variable. This can
+be done by commenting out the `Xms` and `Xmx` settings in the
+<> file and setting these values via `ES_JAVA_OPTS`:
+
+[source,sh]
+------------------
+ES_JAVA_OPTS="-Xms2g -Xmx2g" ./bin/elasticsearch <1>
+ES_JAVA_OPTS="-Xms4000m -Xmx4000m" ./bin/elasticsearch <2>
+------------------
+<1> Set the minimum and maximum heap size to 2 GB.
+<2> Set the minimum and maximum heap size to 4000 MB.
+
+NOTE: Configuring the heap for the <> is
+different than the above. The values initially populated for the Windows service
+can be configured as above but are different after the service has been
+installed. Consult the <> for
+additional details.
diff --git a/docs/reference/setup/important-settings/network-host.asciidoc b/docs/reference/setup/important-settings/network-host.asciidoc
new file mode 100644
index 00000000000..7e29e73123d
--- /dev/null
+++ b/docs/reference/setup/important-settings/network-host.asciidoc
@@ -0,0 +1,29 @@
+[[network.host]]
+=== `network.host`
+
+By default, Elasticsearch binds to loopback addresses only -- e.g. `127.0.0.1`
+and `[::1]`. This is sufficient to run a single development node on a server.
+
+TIP: In fact, more than one node can be started from the same `$ES_HOME`
+location on a single node.  This can be useful for testing Elasticsearch's
+ability to form clusters, but it is not a configuration recommended for
+production.
+
+In order to communicate and to form a cluster with nodes on other servers, your
+node will need to bind to a non-loopback address. While there are many
+<>, usually all you need to configure is
+`network.host`:
+
+[source,yaml]
+--------------------------------------------------
+network.host: 192.168.1.10
+--------------------------------------------------
+
+The `network.host` setting also understands some special values such as
+`_local_`, `_site_`, `_global_` and modifiers like `:ip4` and `:ip6`, details of
+which can be found in <>.
+
+IMPORTANT: As soon you provide a custom setting for `network.host`,
+Elasticsearch assumes that you are moving from development mode to production
+mode, and upgrades a number of system startup checks from warnings to
+exceptions. See <> for more information.
diff --git a/docs/reference/setup/important-settings/node-name.asciidoc b/docs/reference/setup/important-settings/node-name.asciidoc
new file mode 100644
index 00000000000..9f7a330662f
--- /dev/null
+++ b/docs/reference/setup/important-settings/node-name.asciidoc
@@ -0,0 +1,22 @@
+[[node.name]]
+=== `node.name`
+
+By default, Elasticsearch will take the 7 first character of the randomly
+generated uuid used as the node id.  Note that the node id is persisted and does
+not change when a node restarts and therefore the default node name will also
+not change.
+
+It is worth configuring a more meaningful name which will also have the
+advantage of persisting after restarting the node:
+
+[source,yaml]
+--------------------------------------------------
+node.name: prod-data-2
+--------------------------------------------------
+
+The `node.name` can also be set to the server's HOSTNAME as follows:
+
+[source,yaml]
+--------------------------------------------------
+node.name: ${HOSTNAME}
+--------------------------------------------------
\ No newline at end of file
diff --git a/docs/reference/setup/important-settings/path-settings.asciidoc b/docs/reference/setup/important-settings/path-settings.asciidoc
new file mode 100644
index 00000000000..656df2a0718
--- /dev/null
+++ b/docs/reference/setup/important-settings/path-settings.asciidoc
@@ -0,0 +1,32 @@
+[[path-settings]]
+=== `path.data` and `path.logs`
+
+If you are using the `.zip` or `.tar.gz` archives, the `data` and `logs`
+directories are sub-folders of `$ES_HOME`.  If these important folders are left
+in their default locations, there is a high risk of them being deleted while
+upgrading Elasticsearch to a new version.
+
+In production use, you will almost certainly want to change the locations of the
+data and log folder:
+
+[source,yaml]
+--------------------------------------------------
+path:
+  logs: /var/log/elasticsearch
+  data: /var/data/elasticsearch
+--------------------------------------------------
+
+The RPM and Debian distributions already use custom paths for `data` and `logs`.
+
+The `path.data` settings can be set to multiple paths, in which case all paths
+will be used to store data (although the files belonging to a single shard will
+all be stored on the same data path):
+
+[source,yaml]
+--------------------------------------------------
+path:
+  data:
+    - /mnt/elasticsearch_1
+    - /mnt/elasticsearch_2
+    - /mnt/elasticsearch_3
+--------------------------------------------------
\ No newline at end of file
diff --git a/docs/reference/setup/jvm-options.asciidoc b/docs/reference/setup/jvm-options.asciidoc
new file mode 100644
index 00000000000..496824c12b6
--- /dev/null
+++ b/docs/reference/setup/jvm-options.asciidoc
@@ -0,0 +1,82 @@
+[[jvm-options]]
+=== Setting JVM options
+
+The preferred method of setting JVM options (including system properties and JVM
+flags) is via the `jvm.options` configuration file. The default location of this
+file is `config/jvm.options` (when installing from the tar or zip distributions)
+and `/etc/elasticsearch/jvm.options` (when installing from the Debian or RPM
+packages).
+
+This file contains a line-delimited list of JVM arguments following
+a special syntax:
+
+* lines consisting of whitespace only are ignored
+* lines beginning with `#` are treated as comments and are ignored
++
+[source,text]
+-------------------------------------
+# this is a comment
+-------------------------------------
+
+* lines beginning with a `-` are treated as a JVM option that applies
+  independent of the version of the JVM
++
+[source,text]
+-------------------------------------
+-Xmx2g
+-------------------------------------
+
+* lines beginning with a number followed by a `:` followed by a `-` are treated
+  as a JVM option that applies only if the version of the JVM matches the number
++
+[source,text]
+-------------------------------------
+8:-Xmx2g
+-------------------------------------
+
+* lines beginning with a number followed by a `-` followed by a `:` are treated
+  as a JVM option that applies only if the version of the JVM is greater than or
+  equal to the number
++
+[source,text]
+-------------------------------------
+8-:-Xmx2g
+-------------------------------------
+
+* lines beginning with a number followed by a `-` followed by a number followed
+  by a `:` are treated as a JVM option that applies only if the version of the
+  JVM falls in the range of the two numbers
++
+[source,text]
+-------------------------------------
+8-9:-Xmx2g
+-------------------------------------
+
+* all other lines are rejected
+
+You can add custom JVM flags to this file and check this configuration into your
+version control system.
+
+An alternative mechanism for setting Java Virtual Machine options is via the
+`ES_JAVA_OPTS` environment variable. For instance:
+
+[source,sh]
+---------------------------------
+export ES_JAVA_OPTS="$ES_JAVA_OPTS -Djava.io.tmpdir=/path/to/temp/dir"
+./bin/elasticsearch
+---------------------------------
+
+When using the RPM or Debian packages, `ES_JAVA_OPTS` can be specified in the
+<>.
+
+The JVM has a built-in mechanism for observing the `JAVA_TOOL_OPTIONS`
+environment variable. We intentionally ignore this environment variable in our
+packaging scripts. The primary reason for this is that on some OS (e.g., Ubuntu)
+there are agents installed by default via this environment variable that we do
+not want interfering with Elasticsearch.
+
+Additionally, some other Java programs support the `JAVA_OPTS` environment
+variable. This is *not* a mechanism built into the JVM but instead a convention
+in the ecosystem. However, we do not support this environment variable, instead
+supporting setting JVM options via the `jvm.options` file or the environment
+variable `ES_JAVA_OPTS` as above.
diff --git a/docs/reference/setup/sysconfig.asciidoc b/docs/reference/setup/sysconfig.asciidoc
index 5dc015f1451..971d62a9d4d 100644
--- a/docs/reference/setup/sysconfig.asciidoc
+++ b/docs/reference/setup/sysconfig.asciidoc
@@ -6,13 +6,13 @@ resources available to it.  In order to do so, you need to configure your
 operating system to allow the user running Elasticsearch to access more
 resources than allowed by default.
 
-The following settings *must* be addressed before going to production:
+The following settings *must* be considered before going to production:
 
-* <>
 * <>
 * <>
 * <>
 * <>
+* <>
 
 [[dev-vs-prod]]
 [float]
@@ -31,8 +31,6 @@ lose data because of a malconfigured server.
 
 include::sysconfig/configuring.asciidoc[]
 
-include::sysconfig/heap_size.asciidoc[]
-
 include::sysconfig/swap.asciidoc[]
 
 include::sysconfig/file-descriptors.asciidoc[]
diff --git a/docs/reference/setup/sysconfig/configuring.asciidoc b/docs/reference/setup/sysconfig/configuring.asciidoc
index 6f61e5f22cd..8390991aeb1 100644
--- a/docs/reference/setup/sysconfig/configuring.asciidoc
+++ b/docs/reference/setup/sysconfig/configuring.asciidoc
@@ -36,7 +36,6 @@ The new limit is only applied during the current session.
 
 You can consult all currently applied limits with `ulimit -a`.
 
-
 [[limits.conf]]
 ==== `/etc/security/limits.conf`
 
@@ -66,7 +65,6 @@ following line:
 --------------------------------
 ===============================
 
-
 [[sysconfig]]
 ==== Sysconfig file
 
@@ -81,7 +79,6 @@ Debian::  `/etc/default/elasticsearch`
 However, for systems which uses `systemd`, system limits need to be specified
 via <>.
 
-
 [[systemd]]
 ==== Systemd configuration
 
@@ -110,57 +107,3 @@ Once finished, run the following command to reload units:
 ---------------------------------
 sudo systemctl daemon-reload
 ---------------------------------
-
-[[jvm-options]]
-==== Setting JVM options
-
-The preferred method of setting Java Virtual Machine options (including
-system properties and JVM flags) is via the `jvm.options` configuration
-file. The default location of this file is `config/jvm.options` (when
-installing from the tar or zip distributions) and
-`/etc/elasticsearch/jvm.options` (when installing from the Debian or RPM
-packages). This file contains a line-delimited list of JVM arguments following
-a special syntax:
- - lines beginning with `#` are treated as comments and are ignored
- - lines consisting only of whitespace are ignored
- - lines beginning with a `-` are treated as a JVM option that applies
-   independent of the version of the JVM
- - lines beginning with a number followed by a `:` followed by a `-` are treated
-   as a JVM option that applies only if the version of the JVM matches the
-   number
- - lines beginning with a number followed by a `-` followed by a `:` are treated
-   as a JVM option that applies only if the version of the JVM is greater than
-   or equal to the number
- - lines beginning with a number followed by a `-` followed by a `:` followed by
-   a `-` followed by a number are treated as a JVM option that applies only if
-   the version of the JVM falls in the range of the two numbers
- - all other lines are rejected
-
-
-You can add custom JVM flags to this file and
-check this configuration into your version control system.
-
-An alternative mechanism for setting Java Virtual Machine options is
-via the `ES_JAVA_OPTS` environment variable. For instance:
-
-[source,sh]
----------------------------------
-export ES_JAVA_OPTS="$ES_JAVA_OPTS -Djava.io.tmpdir=/path/to/temp/dir"
-./bin/elasticsearch
----------------------------------
-
-When using the RPM or Debian packages, `ES_JAVA_OPTS` can be specified in the
-<>.
-
-The JVM has a built-in mechanism for observing the `JAVA_TOOL_OPTIONS`
-environment variable. We intentionally ignore this environment variable in our
-packaging scripts. The primary reason for this is that on some OS (e.g., Ubuntu)
-there are agents installed by default via this environment variable that we do
-not want interfering with Elasticsearch.
-
-Additionally, some other Java programs support the `JAVA_OPTS` environment
-variable. This is *not* a mechanism built into the JVM but instead a convention
-in the ecosystem. However, we do not support this environment variable, instead
-supporting setting JVM options via the `jvm.options` file or the environment
-variable `ES_JAVA_OPTS` as above.
-
diff --git a/docs/reference/setup/sysconfig/heap_size.asciidoc b/docs/reference/setup/sysconfig/heap_size.asciidoc
deleted file mode 100644
index 96e46709fb0..00000000000
--- a/docs/reference/setup/sysconfig/heap_size.asciidoc
+++ /dev/null
@@ -1,74 +0,0 @@
-[[heap-size]]
-=== Set JVM heap size via jvm.options
-
-By default, Elasticsearch tells the JVM to use a heap with a minimum
-and maximum size of 1 GB. When moving to production, it is
-important to configure heap size to ensure that Elasticsearch has enough
-heap available.
-
-Elasticsearch will assign the entire heap specified in <>
-via the Xms (minimum heap size) and Xmx (maximum heap size) settings.
-
-The value for these setting depends on the amount of RAM available on
-your server. Good rules of thumb are:
-
-* Set the minimum heap size (Xms) and maximum heap size (Xmx) to be
-  equal to each other.
-
-* The more heap available to Elasticsearch, the more memory it can use for
-  caching. But note that too much heap can subject you to long garbage
-  collection pauses.
-
-* Set Xmx to no more than 50% of your physical RAM, to ensure that there
-  is enough physical RAM left for kernel file system caches.
-
-* Don’t set Xmx to above the cutoff that the JVM uses for compressed
-  object pointers (compressed oops); the exact cutoff varies but is
-  near 32 GB. You can verify that you are under the limit by looking
-  for a line in the logs like the following:
-+
-    heap size [1.9gb], compressed ordinary object pointers [true]
-
-* Even better, try to stay below the threshold for zero-based
-  compressed oops; the exact cutoff varies but 26 GB is safe on most
-  systems, but can be as large as 30 GB on some systems. You can verify
-  that you are under the limit by starting Elasticsearch with the JVM
-  options `-XX:+UnlockDiagnosticVMOptions -XX:+PrintCompressedOopsMode`
-  and looking for a line like the following:
-+
---
-    heap address: 0x000000011be00000, size: 27648 MB, zero based Compressed Oops
-
-showing that zero-based compressed oops are enabled instead of
-
-    heap address: 0x0000000118400000, size: 28672 MB, Compressed Oops with base: 0x00000001183ff000
---
-
-Here are examples of how to set the heap size via the jvm.options file:
-
-[source,txt]
-------------------
--Xms2g <1>
--Xmx2g <2>
-------------------
-<1> Set the minimum heap size to 2g.
-<2> Set the maximum heap size to 2g.
-
-It is also possible to set the heap size via an environment variable.
-This can be done by commenting out the `Xms` and `Xmx` settings
-in the jvm.options file and setting these values via `ES_JAVA_OPTS`:
-
-[source,sh]
-------------------
-ES_JAVA_OPTS="-Xms2g -Xmx2g" ./bin/elasticsearch <1>
-ES_JAVA_OPTS="-Xms4000m -Xmx4000m" ./bin/elasticsearch <2>
-------------------
-<1> Set the minimum and maximum heap size to 2 GB.
-<2> Set the minimum and maximum heap size to 4000 MB.
-
-NOTE: Configuring the heap for the <>
-is different than the above. The values initially populated for the
-Windows service can be configured as above but are different after the
-service has been installed. Consult the
-<> for additional
-details.
diff --git a/docs/reference/setup/sysconfig/swap.asciidoc b/docs/reference/setup/sysconfig/swap.asciidoc
index df9ad7babaf..43ec9f44eb5 100644
--- a/docs/reference/setup/sysconfig/swap.asciidoc
+++ b/docs/reference/setup/sysconfig/swap.asciidoc
@@ -42,7 +42,7 @@ Another option available on Linux systems is to ensure that the sysctl value
 should not lead to swapping under normal circumstances, while still allowing the
 whole system to swap in emergency conditions.
 
-[[mlockall]]
+[[bootstrap-memory_lock]]
 ==== Enable `bootstrap.memory_lock`
 
 Another option is to use

From ca70ca669814a6964e55d11b8f53d8face2533f4 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Tue, 12 Dec 2017 10:55:24 -0500
Subject: [PATCH 248/297] Fix BWC release tests

When running the release tests, we set build.snapshot to false and this
causes all version numbers to not have "-SNAPSHOT". This is true even
for the tips of the branches (e.g., currently 5.6.6 on the 5.6
branch). Yet, if we do not set snapshot to false, then we would still be
trying to find artifacts with "-SNAPSHOT" appended which would not have
been build since build.snapshot is false. To fix this, we have to push
build.snapshot into the version logic.

Relates #27778
---
 .../gradle/VersionCollection.groovy              | 16 +++++++---------
 distribution/bwc/build.gradle                    |  2 +-
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy
index bd98288cde1..a6c151ee592 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy
@@ -29,6 +29,7 @@ import java.util.regex.Matcher
 class VersionCollection {
 
     private final List versions
+    private final boolean buildSnapshot = System.getProperty("build.snapshot", "true") == "true"
 
     /**
      * Construct a VersionCollection from the lines of the Version.java file.
@@ -63,7 +64,10 @@ class VersionCollection {
             throw new GradleException("Unexpectedly found no version constants in Versions.java");
         }
 
-        // The tip of each minor series (>= 5.6) is unreleased, so set their 'snapshot' flags
+        /*
+         * The tip of each minor series (>= 5.6) is unreleased, so they must be built from source (we set branch to non-null), and we set
+         * the snapshot flag if and only if build.snapshot is true.
+         */
         Version prevConsideredVersion = null
         boolean found6xSnapshot = false
         for (final int versionIndex = versions.size() - 1; versionIndex >= 0; versionIndex--) {
@@ -85,7 +89,7 @@ class VersionCollection {
 
                 versions[versionIndex] = new Version(
                         currConsideredVersion.major, currConsideredVersion.minor,
-                        currConsideredVersion.revision, currConsideredVersion.suffix, true, branch)
+                        currConsideredVersion.revision, currConsideredVersion.suffix, buildSnapshot, branch)
             }
 
             if (currConsideredVersion.onOrBefore("5.6.0")) {
@@ -95,12 +99,6 @@ class VersionCollection {
             prevConsideredVersion = currConsideredVersion
         }
 
-        // If we're making a release build then the current should not be a snapshot after all.
-        final boolean currentIsSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        if (false == currentIsSnapshot) {
-            versions[-1] = new Version(versions[-1].major, versions[-1].minor, versions[-1].revision, versions[-1].suffix, false, null)
-        }
-
         this.versions = Collections.unmodifiableList(versions)
     }
 
@@ -137,7 +135,7 @@ class VersionCollection {
     private Version getLastSnapshotWithMajor(int targetMajor) {
         final String currentVersion = currentVersion.toString()
         final int snapshotIndex = versions.findLastIndexOf {
-            it.major == targetMajor && it.before(currentVersion) && it.snapshot
+            it.major == targetMajor && it.before(currentVersion) && it.snapshot == buildSnapshot
         }
         return snapshotIndex == -1 ? null : versions[snapshotIndex]
     }
diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle
index 49d320ef676..a2e88dc38a5 100644
--- a/distribution/bwc/build.gradle
+++ b/distribution/bwc/build.gradle
@@ -119,7 +119,7 @@ if (project.hasProperty('bwcVersion')) {
     dependsOn checkoutBwcBranch, writeBuildMetadata
     dir = checkoutDir
     tasks = [':distribution:deb:assemble', ':distribution:rpm:assemble', ':distribution:zip:assemble']
-    startParameter.systemPropertiesArgs = ['build.snapshot': 'true']
+    startParameter.systemPropertiesArgs = ['build.snapshot': System.getProperty("build.snapshot") ?: "true"]
     doLast {
       List missing = [bwcDeb, bwcRpm, bwcZip].grep { file ->
         false == file.exists() }

From 2994366195184d22d609e4a3de871b5b860950f6 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Tue, 12 Dec 2017 11:08:32 -0500
Subject: [PATCH 249/297] Reinsert paragraph about heap size

This paragraph was accidentally dropped when preparing to merge
setting the heap size.
---
 docs/reference/setup/jvm-options.asciidoc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/reference/setup/jvm-options.asciidoc b/docs/reference/setup/jvm-options.asciidoc
index 496824c12b6..bd39b6781dd 100644
--- a/docs/reference/setup/jvm-options.asciidoc
+++ b/docs/reference/setup/jvm-options.asciidoc
@@ -1,6 +1,10 @@
 [[jvm-options]]
 === Setting JVM options
 
+You should rarely need to change Java Virtual Machine (JVM) options. If you do,
+the most likely change is setting the <>. The remainder of
+this document explains in detail how to set JVM options.
+
 The preferred method of setting JVM options (including system properties and JVM
 flags) is via the `jvm.options` configuration file. The default location of this
 file is `config/jvm.options` (when installing from the tar or zip distributions)

From cc1a301b5ec02256b3cbd99324222dc18315f98e Mon Sep 17 00:00:00 2001
From: Nik Everett 
Date: Tue, 12 Dec 2017 11:07:33 -0500
Subject: [PATCH 250/297] Packaging test: add guard for too many files

If you assert that a pattern of files exists but it matches more then
one file the "assert this file exists" code failed with a misleading
error message. This tests if the patter resolved to multiple files and
prints a better error message if it did.
---
 qa/vagrant/src/test/resources/packaging/utils/utils.bash | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/qa/vagrant/src/test/resources/packaging/utils/utils.bash b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
index c43408b3d24..33d8afb09a0 100644
--- a/qa/vagrant/src/test/resources/packaging/utils/utils.bash
+++ b/qa/vagrant/src/test/resources/packaging/utils/utils.bash
@@ -139,6 +139,11 @@ skip_not_zip() {
 
 assert_file_exist() {
     local file="$1"
+    local count=$(echo "$file" | wc -l)
+    [[ "$count" == "1" ]] || {
+      echo "assert_file_exist must be run on a single file at a time but was called on [$count] files: $file"
+      false
+    }
     if [ ! -e "$file" ]; then
         echo "Should exist: ${file} but does not"
     fi

From 33bcfddb54ffad54f11dc16e9a7a213818497421 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= 
Date: Mon, 11 Dec 2017 16:03:59 +0100
Subject: [PATCH 251/297] Use SPI to provide named XContent parsers for ranking
 evaluation

---
 modules/rank-eval/build.gradle                |  1 +
 .../RankEvalNamedXContentProvider.java        | 42 +++++++++++++++++++
 .../index/rankeval/RankEvalPlugin.java        | 14 ++-----
 ...icsearch.plugins.spi.NamedXContentProvider |  1 +
 4 files changed, 47 insertions(+), 11 deletions(-)
 create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java
 create mode 100644 modules/rank-eval/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider

diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle
index 2c18fc40a08..f0479f6e4ab 100644
--- a/modules/rank-eval/build.gradle
+++ b/modules/rank-eval/build.gradle
@@ -20,5 +20,6 @@
 esplugin {
   description 'The Rank Eval module adds APIs to evaluate ranking quality.'
   classname 'org.elasticsearch.index.rankeval.RankEvalPlugin'
+  hasClientJar = true
 }
 
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java
new file mode 100644
index 00000000000..ba241248a02
--- /dev/null
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.rankeval;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RankEvalNamedXContentProvider implements NamedXContentProvider {
+
+    @Override
+    public List getNamedXContentParsers() {
+        List namedXContent = new ArrayList<>();
+        namedXContent.add(
+                new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(PrecisionAtK.NAME), PrecisionAtK::fromXContent));
+        namedXContent.add(new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(MeanReciprocalRank.NAME),
+                MeanReciprocalRank::fromXContent));
+        namedXContent.add(new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(DiscountedCumulativeGain.NAME),
+                DiscountedCumulativeGain::fromXContent));
+        return namedXContent;
+    }
+}
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java
index 278b7f1a613..6b3a23d07d5 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java
@@ -23,13 +23,12 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry.Entry;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
@@ -68,14 +67,7 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin {
     }
 
     @Override
-    public List getNamedXContent() {
-        List namedXContent = new ArrayList<>();
-        namedXContent.add(
-                new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(PrecisionAtK.NAME), PrecisionAtK::fromXContent));
-        namedXContent.add(new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(MeanReciprocalRank.NAME),
-                MeanReciprocalRank::fromXContent));
-        namedXContent.add(new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(DiscountedCumulativeGain.NAME),
-                DiscountedCumulativeGain::fromXContent));
-        return namedXContent;
+    public List getNamedXContent() {
+        return new RankEvalNamedXContentProvider().getNamedXContentParsers();
     }
 }
diff --git a/modules/rank-eval/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider b/modules/rank-eval/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
new file mode 100644
index 00000000000..cd505a43679
--- /dev/null
+++ b/modules/rank-eval/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
@@ -0,0 +1 @@
+org.elasticsearch.index.rankeval.RankEvalNamedXContentProvider
\ No newline at end of file

From 57fc705d5e87d3cca563a0e54d85b03331288ecf Mon Sep 17 00:00:00 2001
From: Nhat Nguyen 
Date: Tue, 12 Dec 2017 19:20:08 -0500
Subject: [PATCH 252/297] Keep commits and translog up to the global checkpoint
 (#27606)

We need to keep index commits and translog operations up to the current
global checkpoint to allow us to throw away unsafe operations and
increase the operation-based recovery chance. This is achieved by a new
index deletion policy.

Relates #10708
---
 .../index/engine/CombinedDeletionPolicy.java  |  87 +++++---
 .../index/engine/InternalEngine.java          |  59 +++---
 .../index/translog/Translog.java              |   4 +-
 .../translog/TranslogDeletionPolicy.java      |  30 ++-
 .../engine/CombinedDeletionPolicyTests.java   | 186 ++++++++++++++----
 .../index/engine/InternalEngineTests.java     | 123 +++++++++++-
 .../index/shard/IndexShardTests.java          |  13 +-
 .../index/translog/SnapshotMatchers.java      |  50 +++++
 .../translog/TranslogDeletionPolicyTests.java |   3 +
 .../index/translog/TranslogTests.java         |  67 +++++--
 .../indices/recovery/RecoveryTests.java       |  45 +++--
 .../org/elasticsearch/test/ESTestCase.java    |   7 +
 12 files changed, 543 insertions(+), 131 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java b/core/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
index add4a443903..2d46ae1d8a8 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
@@ -21,43 +21,48 @@ package org.elasticsearch.index.engine;
 
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexDeletionPolicy;
-import org.apache.lucene.index.SnapshotDeletionPolicy;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogDeletionPolicy;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
+import java.util.function.LongSupplier;
 
 /**
  * An {@link IndexDeletionPolicy} that coordinates between Lucene's commits and the retention of translog generation files,
  * making sure that all translog files that are needed to recover from the Lucene commit are not deleted.
+ * 
      
+ * In particular, this policy will delete index commits whose max sequence number is at most
+ * the current global checkpoint except the index commit which has the highest max sequence number among those.
  */
-class CombinedDeletionPolicy extends IndexDeletionPolicy {
-
+final class CombinedDeletionPolicy extends IndexDeletionPolicy {
     private final TranslogDeletionPolicy translogDeletionPolicy;
     private final EngineConfig.OpenMode openMode;
+    private final LongSupplier globalCheckpointSupplier;
 
-    private final SnapshotDeletionPolicy indexDeletionPolicy;
-
-    CombinedDeletionPolicy(SnapshotDeletionPolicy indexDeletionPolicy, TranslogDeletionPolicy translogDeletionPolicy,
-                           EngineConfig.OpenMode openMode) {
-        this.indexDeletionPolicy = indexDeletionPolicy;
-        this.translogDeletionPolicy = translogDeletionPolicy;
+    CombinedDeletionPolicy(EngineConfig.OpenMode openMode, TranslogDeletionPolicy translogDeletionPolicy,
+                           LongSupplier globalCheckpointSupplier) {
         this.openMode = openMode;
+        this.translogDeletionPolicy = translogDeletionPolicy;
+        this.globalCheckpointSupplier = globalCheckpointSupplier;
     }
 
     @Override
     public void onInit(List commits) throws IOException {
-        indexDeletionPolicy.onInit(commits);
         switch (openMode) {
             case CREATE_INDEX_AND_TRANSLOG:
+                assert commits.isEmpty() : "index is created, but we have commits";
                 break;
             case OPEN_INDEX_CREATE_TRANSLOG:
                 assert commits.isEmpty() == false : "index is opened, but we have no commits";
+                // When an engine starts with OPEN_INDEX_CREATE_TRANSLOG, a new fresh index commit will be created immediately.
+                // We therefore can simply skip processing here as `onCommit` will be called right after with a new commit.
                 break;
             case OPEN_INDEX_AND_TRANSLOG:
                 assert commits.isEmpty() == false : "index is opened, but we have no commits";
-                setLastCommittedTranslogGeneration(commits);
+                onCommit(commits);
                 break;
             default:
                 throw new IllegalArgumentException("unknown openMode [" + openMode + "]");
@@ -66,24 +71,56 @@ class CombinedDeletionPolicy extends IndexDeletionPolicy {
 
     @Override
     public void onCommit(List commits) throws IOException {
-        indexDeletionPolicy.onCommit(commits);
-        setLastCommittedTranslogGeneration(commits);
+        final int keptPosition = indexOfKeptCommits(commits);
+        for (int i = 0; i < keptPosition; i++) {
+            commits.get(i).delete();
+        }
+        updateTranslogDeletionPolicy(commits.get(keptPosition), commits.get(commits.size() - 1));
     }
 
-    private void setLastCommittedTranslogGeneration(List commits) throws IOException {
-        // when opening an existing lucene index, we currently always open the last commit.
-        // we therefore use the translog gen as the one that will be required for recovery
-        final IndexCommit indexCommit = commits.get(commits.size() - 1);
-        assert indexCommit.isDeleted() == false : "last commit is deleted";
-        long minGen = Long.parseLong(indexCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY));
-        translogDeletionPolicy.setMinTranslogGenerationForRecovery(minGen);
+    private void updateTranslogDeletionPolicy(final IndexCommit minRequiredCommit, final IndexCommit lastCommit) throws IOException {
+        assert minRequiredCommit.isDeleted() == false : "The minimum required commit must not be deleted";
+        final long minRequiredGen = Long.parseLong(minRequiredCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY));
+
+        assert lastCommit.isDeleted() == false : "The last commit must not be deleted";
+        final long lastGen = Long.parseLong(lastCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY));
+
+        assert minRequiredGen <= lastGen : "minRequiredGen must not be greater than lastGen";
+        translogDeletionPolicy.setTranslogGenerationOfLastCommit(lastGen);
+        translogDeletionPolicy.setMinTranslogGenerationForRecovery(minRequiredGen);
     }
 
-    public SnapshotDeletionPolicy getIndexDeletionPolicy() {
-        return indexDeletionPolicy;
-    }
+    /**
+     * Find the highest index position of a safe index commit whose max sequence number is not greater than the global checkpoint.
+     * Index commits with different translog UUID will be filtered out as they don't belong to this engine.
+     */
+    private int indexOfKeptCommits(List commits) throws IOException {
+        final long currentGlobalCheckpoint = globalCheckpointSupplier.getAsLong();
+        final String expectedTranslogUUID = commits.get(commits.size() - 1).getUserData().get(Translog.TRANSLOG_UUID_KEY);
 
-    public TranslogDeletionPolicy getTranslogDeletionPolicy() {
-        return translogDeletionPolicy;
+        // Commits are sorted by age (the 0th one is the oldest commit).
+        for (int i = commits.size() - 1; i >= 0; i--) {
+            final Map commitUserData = commits.get(i).getUserData();
+            // Ignore index commits with different translog uuid.
+            if (expectedTranslogUUID.equals(commitUserData.get(Translog.TRANSLOG_UUID_KEY)) == false) {
+                return i + 1;
+            }
+            // 5.x commits do not contain MAX_SEQ_NO.
+            if (commitUserData.containsKey(SequenceNumbers.MAX_SEQ_NO) == false) {
+                return i;
+            }
+            final long maxSeqNoFromCommit = Long.parseLong(commitUserData.get(SequenceNumbers.MAX_SEQ_NO));
+            if (maxSeqNoFromCommit <= currentGlobalCheckpoint) {
+                return i;
+            }
+        }
+        /*
+         * We may reach to this point in these cases:
+         * 1. In the previous 6.x, we keep only the last commit - which is likely not a safe commit if writes are in progress.
+         * Thus, after upgrading, we may not find a safe commit until we can reserve one.
+         * 2. In peer-recovery, if the file-based happens, a replica will be received the latest commit from a primary.
+         * However, that commit may not be a safe commit if writes are in progress in the primary.
+         */
+        return 0;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 47056c3b010..e4f6a6f9b0a 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LiveIndexWriterConfig;
 import org.apache.lucene.index.MergePolicy;
@@ -128,7 +127,7 @@ public class InternalEngine extends Engine {
 
     private final String uidField;
 
-    private final CombinedDeletionPolicy deletionPolicy;
+    private final SnapshotDeletionPolicy snapshotDeletionPolicy;
 
     // How many callers are currently requesting index throttling.  Currently there are only two situations where we do this: when merges
     // are falling behind and when writing indexing buffer to disk is too slow.  When this is 0, there is no throttling, else we throttling
@@ -167,8 +166,6 @@ public class InternalEngine extends Engine {
                 engineConfig.getIndexSettings().getTranslogRetentionSize().getBytes(),
                 engineConfig.getIndexSettings().getTranslogRetentionAge().getMillis()
         );
-        this.deletionPolicy = new CombinedDeletionPolicy(
-                new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()), translogDeletionPolicy, openMode);
         store.incRef();
         IndexWriter writer = null;
         Translog translog = null;
@@ -182,30 +179,19 @@ public class InternalEngine extends Engine {
             mergeScheduler = scheduler = new EngineMergeScheduler(engineConfig.getShardId(), engineConfig.getIndexSettings());
             throttle = new IndexThrottle();
             try {
-                final SeqNoStats seqNoStats;
-                switch (openMode) {
-                    case OPEN_INDEX_AND_TRANSLOG:
-                        writer = createWriter(false);
-                        final long globalCheckpoint = Translog.readGlobalCheckpoint(engineConfig.getTranslogConfig().getTranslogPath());
-                        seqNoStats = store.loadSeqNoStats(globalCheckpoint);
-                        break;
-                    case OPEN_INDEX_CREATE_TRANSLOG:
-                        writer = createWriter(false);
-                        seqNoStats = store.loadSeqNoStats(SequenceNumbers.UNASSIGNED_SEQ_NO);
-                        break;
-                    case CREATE_INDEX_AND_TRANSLOG:
-                        writer = createWriter(true);
-                        seqNoStats = new SeqNoStats(
-                                SequenceNumbers.NO_OPS_PERFORMED,
-                                SequenceNumbers.NO_OPS_PERFORMED,
-                                SequenceNumbers.UNASSIGNED_SEQ_NO);
-                        break;
-                    default:
-                        throw new IllegalArgumentException(openMode.toString());
-                }
+                final SeqNoStats seqNoStats = loadSeqNoStats(openMode);
                 logger.trace("recovered [{}]", seqNoStats);
-                seqNoService = seqNoServiceSupplier.apply(engineConfig, seqNoStats);
+                this.seqNoService = seqNoServiceSupplier.apply(engineConfig, seqNoStats);
+                this.snapshotDeletionPolicy = new SnapshotDeletionPolicy(
+                    new CombinedDeletionPolicy(openMode, translogDeletionPolicy, seqNoService::getGlobalCheckpoint)
+                );
+                writer = createWriter(openMode == EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG);
                 updateMaxUnsafeAutoIdTimestampFromWriter(writer);
+                assert engineConfig.getForceNewHistoryUUID() == false
+                    || openMode == EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG
+                    || openMode == EngineConfig.OpenMode.OPEN_INDEX_CREATE_TRANSLOG
+                    : "OpenMode must be either CREATE_INDEX_AND_TRANSLOG or OPEN_INDEX_CREATE_TRANSLOG if forceNewHistoryUUID; " +
+                    "openMode [" + openMode + "], forceNewHistoryUUID [" + engineConfig.getForceNewHistoryUUID() + "]";
                 historyUUID = loadOrGenerateHistoryUUID(writer, engineConfig.getForceNewHistoryUUID());
                 Objects.requireNonNull(historyUUID, "history uuid should not be null");
                 indexWriter = writer;
@@ -380,6 +366,23 @@ public class InternalEngine extends Engine {
             seqNoStats.getGlobalCheckpoint());
     }
 
+    private SeqNoStats loadSeqNoStats(EngineConfig.OpenMode openMode) throws IOException {
+        switch (openMode) {
+            case OPEN_INDEX_AND_TRANSLOG:
+                final long globalCheckpoint = Translog.readGlobalCheckpoint(engineConfig.getTranslogConfig().getTranslogPath());
+                return store.loadSeqNoStats(globalCheckpoint);
+            case OPEN_INDEX_CREATE_TRANSLOG:
+                return store.loadSeqNoStats(SequenceNumbers.UNASSIGNED_SEQ_NO);
+            case CREATE_INDEX_AND_TRANSLOG:
+                return new SeqNoStats(
+                    SequenceNumbers.NO_OPS_PERFORMED,
+                    SequenceNumbers.NO_OPS_PERFORMED,
+                    SequenceNumbers.UNASSIGNED_SEQ_NO);
+            default:
+                throw new IllegalArgumentException(openMode.toString());
+        }
+    }
+
     @Override
     public InternalEngine recoverFromTranslog() throws IOException {
         flushLock.lock();
@@ -1607,7 +1610,7 @@ public class InternalEngine extends Engine {
         }
         try (ReleasableLock lock = readLock.acquire()) {
             logger.trace("pulling snapshot");
-            return new IndexCommitRef(deletionPolicy.getIndexDeletionPolicy());
+            return new IndexCommitRef(snapshotDeletionPolicy);
         } catch (IOException e) {
             throw new SnapshotFailedEngineException(shardId, e);
         }
@@ -1788,7 +1791,7 @@ public class InternalEngine extends Engine {
         final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer());
         iwc.setCommitOnClose(false); // we by default don't commit on close
         iwc.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND);
-        iwc.setIndexDeletionPolicy(deletionPolicy);
+        iwc.setIndexDeletionPolicy(snapshotDeletionPolicy);
         // with tests.verbose, lucene sets this up: plumb to align with filesystem stream
         boolean verbose = false;
         try {
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index 80033833899..9e2e7ddbd06 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -372,14 +372,14 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
      * Returns the number of operations in the translog files that aren't committed to lucene.
      */
     public int uncommittedOperations() {
-        return totalOperations(deletionPolicy.getMinTranslogGenerationForRecovery());
+        return totalOperations(deletionPolicy.getTranslogGenerationOfLastCommit());
     }
 
     /**
      * Returns the size in bytes of the translog files that aren't committed to lucene.
      */
     public long uncommittedSizeInBytes() {
-        return sizeInBytesByMinGen(deletionPolicy.getMinTranslogGenerationForRecovery());
+        return sizeInBytesByMinGen(deletionPolicy.getTranslogGenerationOfLastCommit());
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogDeletionPolicy.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogDeletionPolicy.java
index adee4bd9fa9..5eba198378a 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogDeletionPolicy.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogDeletionPolicy.java
@@ -54,6 +54,11 @@ public class TranslogDeletionPolicy {
      */
     private long minTranslogGenerationForRecovery = 1;
 
+    /**
+     * This translog generation is used to calculate the number of uncommitted operations since the last index commit.
+     */
+    private long translogGenerationOfLastCommit = 1;
+
     private long retentionSizeInBytes;
 
     private long retentionAgeInMillis;
@@ -69,13 +74,24 @@ public class TranslogDeletionPolicy {
     }
 
     public synchronized void setMinTranslogGenerationForRecovery(long newGen) {
-        if (newGen < minTranslogGenerationForRecovery) {
-            throw new IllegalArgumentException("minTranslogGenerationForRecovery can't go backwards. new [" + newGen + "] current [" +
-                minTranslogGenerationForRecovery + "]");
+        if (newGen < minTranslogGenerationForRecovery || newGen > translogGenerationOfLastCommit) {
+            throw new IllegalArgumentException("Invalid minTranslogGenerationForRecovery can't go backwards; new [" + newGen + "]," +
+                "current [" + minTranslogGenerationForRecovery + "], lastGen [" + translogGenerationOfLastCommit + "]");
         }
         minTranslogGenerationForRecovery = newGen;
     }
 
+    /**
+     * Sets the translog generation of the last index commit.
+     */
+    public synchronized void setTranslogGenerationOfLastCommit(long lastGen) {
+        if (lastGen < translogGenerationOfLastCommit || lastGen < minTranslogGenerationForRecovery) {
+            throw new IllegalArgumentException("Invalid translogGenerationOfLastCommit; new [" + lastGen + "]," +
+                "current [" + translogGenerationOfLastCommit + "], minRequiredGen [" + minTranslogGenerationForRecovery + "]");
+        }
+        translogGenerationOfLastCommit = lastGen;
+    }
+
     public synchronized void setRetentionSizeInBytes(long bytes) {
         retentionSizeInBytes = bytes;
     }
@@ -193,6 +209,14 @@ public class TranslogDeletionPolicy {
         return minTranslogGenerationForRecovery;
     }
 
+    /**
+     * Returns a translog generation that will be used to calculate the number of uncommitted operations since the last index commit.
+     * See {@link Translog#uncommittedOperations()} and {@link Translog#uncommittedSizeInBytes()}
+     */
+    public synchronized long getTranslogGenerationOfLastCommit() {
+        return translogGenerationOfLastCommit;
+    }
+
     synchronized long getTranslogRefCount(long gen) {
         final Counter counter = translogRefCounts.get(gen);
         return counter == null ? 0 : counter.get();
diff --git a/core/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java b/core/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
index 5d4385cbd38..0fc6195161a 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
@@ -19,66 +19,178 @@
 
 package org.elasticsearch.index.engine;
 
+import com.carrotsearch.hppc.LongArrayList;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.SnapshotDeletionPolicy;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogDeletionPolicy;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
 
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.index.engine.EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG;
+import static org.elasticsearch.index.engine.EngineConfig.OpenMode.OPEN_INDEX_CREATE_TRANSLOG;
 import static org.elasticsearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class CombinedDeletionPolicyTests extends ESTestCase {
 
-    public void testPassThrough() throws IOException {
-        SnapshotDeletionPolicy indexDeletionPolicy = mock(SnapshotDeletionPolicy.class);
-        CombinedDeletionPolicy combinedDeletionPolicy = new CombinedDeletionPolicy(indexDeletionPolicy, createTranslogDeletionPolicy(),
-            EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG);
-        List commitList = new ArrayList<>();
-        long count = randomIntBetween(1, 3);
-        for (int i = 0; i < count; i++) {
-            commitList.add(mockIndexCommitWithTranslogGen(randomNonNegativeLong()));
+    public void testKeepCommitsAfterGlobalCheckpoint() throws Exception {
+        final AtomicLong globalCheckpoint = new AtomicLong();
+        TranslogDeletionPolicy translogPolicy = createTranslogDeletionPolicy();
+        CombinedDeletionPolicy indexPolicy = new CombinedDeletionPolicy(OPEN_INDEX_AND_TRANSLOG, translogPolicy, globalCheckpoint::get);
+
+        final LongArrayList maxSeqNoList = new LongArrayList();
+        final LongArrayList translogGenList = new LongArrayList();
+        final List commitList = new ArrayList<>();
+        int totalCommits = between(2, 20);
+        long lastMaxSeqNo = 0;
+        long lastTranslogGen = 0;
+        final UUID translogUUID = UUID.randomUUID();
+        for (int i = 0; i < totalCommits; i++) {
+            lastMaxSeqNo += between(1, 10000);
+            lastTranslogGen += between(1, 100);
+            commitList.add(mockIndexCommit(lastMaxSeqNo, translogUUID, lastTranslogGen));
+            maxSeqNoList.add(lastMaxSeqNo);
+            translogGenList.add(lastTranslogGen);
         }
-        combinedDeletionPolicy.onInit(commitList);
-        verify(indexDeletionPolicy, times(1)).onInit(commitList);
-        combinedDeletionPolicy.onCommit(commitList);
-        verify(indexDeletionPolicy, times(1)).onCommit(commitList);
+
+        int keptIndex = randomInt(commitList.size() - 1);
+        final long lower = maxSeqNoList.get(keptIndex);
+        final long upper = keptIndex == commitList.size() - 1 ?
+            Long.MAX_VALUE : Math.max(maxSeqNoList.get(keptIndex), maxSeqNoList.get(keptIndex + 1) - 1);
+        globalCheckpoint.set(randomLongBetween(lower, upper));
+        indexPolicy.onCommit(commitList);
+
+        for (int i = 0; i < commitList.size(); i++) {
+            if (i < keptIndex) {
+                verify(commitList.get(i), times(1)).delete();
+            } else {
+                verify(commitList.get(i), never()).delete();
+            }
+        }
+        assertThat(translogPolicy.getMinTranslogGenerationForRecovery(), equalTo(translogGenList.get(keptIndex)));
+        assertThat(translogPolicy.getTranslogGenerationOfLastCommit(), equalTo(lastTranslogGen));
     }
 
-    public void testSettingMinTranslogGen() throws IOException {
-        SnapshotDeletionPolicy indexDeletionPolicy = mock(SnapshotDeletionPolicy.class);
-        final TranslogDeletionPolicy translogDeletionPolicy = mock(TranslogDeletionPolicy.class);
-        CombinedDeletionPolicy combinedDeletionPolicy = new CombinedDeletionPolicy(indexDeletionPolicy, translogDeletionPolicy,
-            EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG);
-        List commitList = new ArrayList<>();
-        long count = randomIntBetween(10, 20);
-        long lastGen = 0;
-        for (int i = 0; i < count; i++) {
-            lastGen += randomIntBetween(10, 20000);
-            commitList.add(mockIndexCommitWithTranslogGen(lastGen));
-        }
-        combinedDeletionPolicy.onInit(commitList);
-        verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(lastGen);
-        commitList.clear();
-        for (int i = 0; i < count; i++) {
-            lastGen += randomIntBetween(10, 20000);
-            commitList.add(mockIndexCommitWithTranslogGen(lastGen));
-        }
-        combinedDeletionPolicy.onCommit(commitList);
-        verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(lastGen);
+    public void testIgnoreSnapshottingCommits() throws Exception {
+        final AtomicLong globalCheckpoint = new AtomicLong();
+        final UUID translogUUID = UUID.randomUUID();
+        TranslogDeletionPolicy translogPolicy = createTranslogDeletionPolicy();
+        CombinedDeletionPolicy indexPolicy = new CombinedDeletionPolicy(OPEN_INDEX_AND_TRANSLOG, translogPolicy, globalCheckpoint::get);
+
+        long firstMaxSeqNo = randomLongBetween(0, Long.MAX_VALUE - 1);
+        long secondMaxSeqNo = randomLongBetween(firstMaxSeqNo + 1, Long.MAX_VALUE);
+
+        long lastTranslogGen = randomNonNegativeLong();
+        final IndexCommit firstCommit = mockIndexCommit(firstMaxSeqNo, translogUUID, randomLongBetween(0, lastTranslogGen));
+        final IndexCommit secondCommit = mockIndexCommit(secondMaxSeqNo, translogUUID, lastTranslogGen);
+        SnapshotDeletionPolicy snapshotDeletionPolicy = new SnapshotDeletionPolicy(indexPolicy);
+
+        snapshotDeletionPolicy.onInit(Arrays.asList(firstCommit));
+        snapshotDeletionPolicy.snapshot();
+        assertThat(snapshotDeletionPolicy.getSnapshots(), contains(firstCommit));
+
+        // SnapshotPolicy prevents the first commit from deleting, but CombinedPolicy does not retain its translog.
+        globalCheckpoint.set(randomLongBetween(secondMaxSeqNo, Long.MAX_VALUE));
+        snapshotDeletionPolicy.onCommit(Arrays.asList(firstCommit, secondCommit));
+        verify(firstCommit, never()).delete();
+        verify(secondCommit, never()).delete();
+        assertThat(translogPolicy.getMinTranslogGenerationForRecovery(), equalTo(lastTranslogGen));
+        assertThat(translogPolicy.getTranslogGenerationOfLastCommit(), equalTo(lastTranslogGen));
     }
 
-    IndexCommit mockIndexCommitWithTranslogGen(long gen) throws IOException {
-        IndexCommit commit = mock(IndexCommit.class);
-        when(commit.getUserData()).thenReturn(Collections.singletonMap(Translog.TRANSLOG_GENERATION_KEY, Long.toString(gen)));
+    public void testLegacyIndex() throws Exception {
+        final AtomicLong globalCheckpoint = new AtomicLong();
+        final UUID translogUUID = UUID.randomUUID();
+
+        TranslogDeletionPolicy translogPolicy = createTranslogDeletionPolicy();
+        CombinedDeletionPolicy indexPolicy = new CombinedDeletionPolicy(OPEN_INDEX_AND_TRANSLOG, translogPolicy, globalCheckpoint::get);
+
+        long legacyTranslogGen = randomNonNegativeLong();
+        IndexCommit legacyCommit = mockLegacyIndexCommit(translogUUID, legacyTranslogGen);
+        indexPolicy.onInit(singletonList(legacyCommit));
+        verify(legacyCommit, never()).delete();
+        assertThat(translogPolicy.getMinTranslogGenerationForRecovery(), equalTo(legacyTranslogGen));
+        assertThat(translogPolicy.getTranslogGenerationOfLastCommit(), equalTo(legacyTranslogGen));
+
+        long safeTranslogGen = randomLongBetween(legacyTranslogGen, Long.MAX_VALUE);
+        long maxSeqNo = randomLongBetween(1, Long.MAX_VALUE);
+        final IndexCommit freshCommit = mockIndexCommit(maxSeqNo, translogUUID, safeTranslogGen);
+
+        globalCheckpoint.set(randomLongBetween(0, maxSeqNo - 1));
+        indexPolicy.onCommit(Arrays.asList(legacyCommit, freshCommit));
+        verify(legacyCommit, times(0)).delete();
+        verify(freshCommit, times(0)).delete();
+        assertThat(translogPolicy.getMinTranslogGenerationForRecovery(), equalTo(legacyTranslogGen));
+        assertThat(translogPolicy.getTranslogGenerationOfLastCommit(), equalTo(safeTranslogGen));
+
+        // Make the fresh commit safe.
+        globalCheckpoint.set(randomLongBetween(maxSeqNo, Long.MAX_VALUE));
+        indexPolicy.onCommit(Arrays.asList(legacyCommit, freshCommit));
+        verify(legacyCommit, times(1)).delete();
+        verify(freshCommit, times(0)).delete();
+        assertThat(translogPolicy.getMinTranslogGenerationForRecovery(), equalTo(safeTranslogGen));
+        assertThat(translogPolicy.getTranslogGenerationOfLastCommit(), equalTo(safeTranslogGen));
+    }
+
+    public void testDeleteInvalidCommits() throws Exception {
+        final AtomicLong globalCheckpoint = new AtomicLong(randomNonNegativeLong());
+        TranslogDeletionPolicy translogPolicy = createTranslogDeletionPolicy();
+        CombinedDeletionPolicy indexPolicy = new CombinedDeletionPolicy(OPEN_INDEX_CREATE_TRANSLOG, translogPolicy, globalCheckpoint::get);
+
+        final int invalidCommits = between(1, 10);
+        final List commitList = new ArrayList<>();
+        for (int i = 0; i < invalidCommits; i++) {
+            commitList.add(mockIndexCommit(randomNonNegativeLong(), UUID.randomUUID(), randomNonNegativeLong()));
+        }
+
+        final UUID expectedTranslogUUID = UUID.randomUUID();
+        long lastTranslogGen = 0;
+        final int validCommits = between(1, 10);
+        for (int i = 0; i < validCommits; i++) {
+            lastTranslogGen += between(1, 1000);
+            commitList.add(mockIndexCommit(randomNonNegativeLong(), expectedTranslogUUID, lastTranslogGen));
+        }
+
+        // We should never keep invalid commits regardless of the value of the global checkpoint.
+        indexPolicy.onCommit(commitList);
+        for (int i = 0; i < invalidCommits - 1; i++) {
+            verify(commitList.get(i), times(1)).delete();
+        }
+    }
+
+    IndexCommit mockIndexCommit(long maxSeqNo, UUID translogUUID, long translogGen) throws IOException {
+        final Map userData = new HashMap<>();
+        userData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(maxSeqNo));
+        userData.put(Translog.TRANSLOG_UUID_KEY, translogUUID.toString());
+        userData.put(Translog.TRANSLOG_GENERATION_KEY, Long.toString(translogGen));
+        final IndexCommit commit = mock(IndexCommit.class);
+        when(commit.getUserData()).thenReturn(userData);
+        return commit;
+    }
+
+    IndexCommit mockLegacyIndexCommit(UUID translogUUID, long translogGen) throws IOException {
+        final Map userData = new HashMap<>();
+        userData.put(Translog.TRANSLOG_UUID_KEY, translogUUID.toString());
+        userData.put(Translog.TRANSLOG_GENERATION_KEY, Long.toString(translogGen));
+        final IndexCommit commit = mock(IndexCommit.class);
+        when(commit.getUserData()).thenReturn(userData);
         return commit;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 2d772daf7cc..999b4dca563 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -79,6 +79,7 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver;
@@ -113,6 +114,7 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
 import org.elasticsearch.index.store.DirectoryUtils;
 import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.translog.SnapshotMatchers;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
@@ -165,6 +167,7 @@ import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
@@ -945,17 +948,47 @@ public class InternalEngineTests extends EngineTestCase {
     }
 
     public void testCommitAdvancesMinTranslogForRecovery() throws IOException {
+        IOUtils.close(engine, store);
+        final Path translogPath = createTempDir();
+        store = createStore();
+        final AtomicBoolean inSync = new AtomicBoolean(randomBoolean());
+        final BiFunction seqNoServiceSupplier = (config, seqNoStats) ->
+            new SequenceNumbersService(
+                config.getShardId(),
+                config.getAllocationId(),
+                config.getIndexSettings(),
+                seqNoStats.getMaxSeqNo(),
+                seqNoStats.getLocalCheckpoint(),
+                seqNoStats.getGlobalCheckpoint()) {
+                @Override
+                public long getGlobalCheckpoint() {
+                    return inSync.get() ? getLocalCheckpoint() : SequenceNumbers.UNASSIGNED_SEQ_NO;
+                }
+            };
+        engine = new InternalEngine(config(defaultSettings, store, translogPath, newMergePolicy(), null, null), seqNoServiceSupplier);
         ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null);
         engine.index(indexForDoc(doc));
+
         engine.flush();
         assertThat(engine.getTranslog().currentFileGeneration(), equalTo(2L));
-        assertThat(engine.getTranslog().getDeletionPolicy().getMinTranslogGenerationForRecovery(), equalTo(2L));
+        assertThat(engine.getTranslog().getDeletionPolicy().getMinTranslogGenerationForRecovery(), equalTo(inSync.get() ? 2L : 1L));
+        assertThat(engine.getTranslog().getDeletionPolicy().getTranslogGenerationOfLastCommit(), equalTo(2L));
+
         engine.flush();
         assertThat(engine.getTranslog().currentFileGeneration(), equalTo(2L));
-        assertThat(engine.getTranslog().getDeletionPolicy().getMinTranslogGenerationForRecovery(), equalTo(2L));
+        assertThat(engine.getTranslog().getDeletionPolicy().getMinTranslogGenerationForRecovery(), equalTo(inSync.get() ? 2L : 1L));
+        assertThat(engine.getTranslog().getDeletionPolicy().getTranslogGenerationOfLastCommit(), equalTo(2L));
+
         engine.flush(true, true);
         assertThat(engine.getTranslog().currentFileGeneration(), equalTo(3L));
-        assertThat(engine.getTranslog().getDeletionPolicy().getMinTranslogGenerationForRecovery(), equalTo(3L));
+        assertThat(engine.getTranslog().getDeletionPolicy().getMinTranslogGenerationForRecovery(), equalTo(inSync.get() ? 3L : 1L));
+        assertThat(engine.getTranslog().getDeletionPolicy().getTranslogGenerationOfLastCommit(), equalTo(3L));
+
+        inSync.set(true);
+        engine.flush(true, true);
+        assertThat(engine.getTranslog().currentFileGeneration(), equalTo(4L));
+        assertThat(engine.getTranslog().getDeletionPolicy().getMinTranslogGenerationForRecovery(), equalTo(4L));
+        assertThat(engine.getTranslog().getDeletionPolicy().getTranslogGenerationOfLastCommit(), equalTo(4L));
     }
 
     public void testSyncedFlush() throws IOException {
@@ -2359,10 +2392,26 @@ public class InternalEngineTests extends EngineTestCase {
         );
         indexSettings.updateIndexMetaData(builder.build());
 
+        final BiFunction seqNoServiceSupplier = (config, seqNoStats) ->
+            new SequenceNumbersService(
+                config.getShardId(),
+                config.getAllocationId(),
+                config.getIndexSettings(),
+                seqNoStats.getMaxSeqNo(),
+                seqNoStats.getLocalCheckpoint(),
+                seqNoStats.getGlobalCheckpoint()) {
+                @Override
+                public long getGlobalCheckpoint() {
+                    return getLocalCheckpoint();
+                }
+            };
+
         try (Store store = createStore()) {
             AtomicBoolean throwErrorOnCommit = new AtomicBoolean();
             final Path translogPath = createTempDir();
-            try (InternalEngine engine = new InternalEngine(config(indexSettings, store, translogPath, newMergePolicy(), null, null)) {
+            try (InternalEngine engine =
+                     new InternalEngine(config(indexSettings, store, translogPath, newMergePolicy(), null, null), seqNoServiceSupplier) {
+
                 @Override
                 protected void commitIndexWriter(IndexWriter writer, Translog translog, String syncId) throws IOException {
                     super.commitIndexWriter(writer, translog, syncId);
@@ -2377,7 +2426,8 @@ public class InternalEngineTests extends EngineTestCase {
                 FlushFailedEngineException e = expectThrows(FlushFailedEngineException.class, engine::flush);
                 assertThat(e.getCause().getMessage(), equalTo("power's out"));
             }
-            try (InternalEngine engine = new InternalEngine(config(indexSettings, store, translogPath, newMergePolicy(), null, null))) {
+            try (InternalEngine engine =
+                     new InternalEngine(config(indexSettings, store, translogPath, newMergePolicy(), null, null), seqNoServiceSupplier)) {
                 engine.recoverFromTranslog();
                 assertVisibleCount(engine, 1);
                 final long committedGen = Long.valueOf(
@@ -2608,13 +2658,16 @@ public class InternalEngineTests extends EngineTestCase {
         EngineConfig config = engine.config();
 
         EngineConfig newConfig = new EngineConfig(
-            randomBoolean() ? EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG : EngineConfig.OpenMode.OPEN_INDEX_CREATE_TRANSLOG,
+            randomBoolean() ? EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG : EngineConfig.OpenMode.OPEN_INDEX_CREATE_TRANSLOG,
             shardId, allocationId.getId(),
             threadPool, config.getIndexSettings(), null, store, newMergePolicy(), config.getAnalyzer(), config.getSimilarity(),
             new CodecService(null, logger), config.getEventListener(), IndexSearcher.getDefaultQueryCache(),
             IndexSearcher.getDefaultQueryCachingPolicy(), true, config.getTranslogConfig(), TimeValue.timeValueMinutes(5),
             config.getExternalRefreshListener(), config.getInternalRefreshListener(), null, config.getTranslogRecoveryRunner(),
             new NoneCircuitBreakerService());
+        if (newConfig.getOpenMode() == EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG) {
+            Lucene.cleanLuceneIndex(store.directory());
+        }
         engine = new InternalEngine(newConfig);
         if (newConfig.getOpenMode() == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG) {
             engine.recoverFromTranslog();
@@ -4116,4 +4169,62 @@ public class InternalEngineTests extends EngineTestCase {
         }
     }
 
+    public void testKeepTranslogAfterGlobalCheckpoint() throws Exception {
+        IOUtils.close(engine, store);
+        final AtomicLong globalCheckpoint = new AtomicLong(0);
+        final BiFunction seqNoServiceSupplier = (config, seqNoStats) ->
+            new SequenceNumbersService(
+                config.getShardId(),
+                config.getAllocationId(),
+                config.getIndexSettings(),
+                seqNoStats.getMaxSeqNo(),
+                seqNoStats.getLocalCheckpoint(),
+                seqNoStats.getGlobalCheckpoint()) {
+                @Override
+                public long getGlobalCheckpoint() {
+                    return globalCheckpoint.get();
+                }
+            };
+
+        final IndexSettings indexSettings = new IndexSettings(defaultSettings.getIndexMetaData(), defaultSettings.getNodeSettings(),
+            defaultSettings.getScopedSettings());
+        IndexMetaData.Builder builder = IndexMetaData.builder(indexSettings.getIndexMetaData())
+            .settings(Settings.builder().put(indexSettings.getSettings())
+                .put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomFrom("-1", "100micros", "30m"))
+                .put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), randomFrom("-1", "512b", "1gb")));
+        indexSettings.updateIndexMetaData(builder.build());
+
+        store = createStore();
+        try (InternalEngine engine
+                 = createEngine(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null, seqNoServiceSupplier)) {
+            int numDocs = scaledRandomIntBetween(10, 100);
+            for (int docId = 0; docId < numDocs; docId++) {
+                ParseContext.Document document = testDocumentWithTextField();
+                document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
+                engine.index(indexForDoc(testParsedDocument(Integer.toString(docId), null, document, B_1, null)));
+                if (frequently()) {
+                    globalCheckpoint.set(randomIntBetween(
+                        Math.toIntExact(engine.seqNoService().getGlobalCheckpoint()),
+                        Math.toIntExact(engine.seqNoService().getLocalCheckpoint())));
+                }
+                if (frequently()) {
+                    engine.flush(randomBoolean(), true);
+                    final List commits = DirectoryReader.listCommits(store.directory());
+                    // Keep only one safe commit as the oldest commit.
+                    final IndexCommit safeCommit = commits.get(0);
+                    assertThat(Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)),
+                        lessThanOrEqualTo(globalCheckpoint.get()));
+                    for (int i = 1; i < commits.size(); i++) {
+                        assertThat(Long.parseLong(commits.get(i).getUserData().get(SequenceNumbers.MAX_SEQ_NO)),
+                            greaterThan(globalCheckpoint.get()));
+                    }
+                    // Make sure we keep all translog operations after the local checkpoint of the safe commit.
+                    long localCheckpointFromSafeCommit = Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
+                    try (Translog.Snapshot snapshot = engine.getTranslog().newSnapshot()) {
+                        assertThat(snapshot, SnapshotMatchers.containsSeqNoRange(localCheckpointFromSafeCommit + 1, docId));
+                    }
+                }
+            }
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index e4b1eb083b5..1410d4978b1 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -1051,8 +1051,9 @@ public class IndexShardTests extends IndexShardTestCase {
         closeShards(indexShard);
     }
 
-    public void testAcquireIndexCommit() throws IOException {
-        final IndexShard shard = newStartedShard();
+    public void testAcquireIndexCommit() throws Exception {
+        boolean isPrimary = randomBoolean();
+        final IndexShard shard = newStartedShard(isPrimary);
         int numDocs = randomInt(20);
         for (int i = 0; i < numDocs; i++) {
             indexDoc(shard, "type", "id_" + i);
@@ -1069,6 +1070,14 @@ public class IndexShardTests extends IndexShardTestCase {
             assertThat(reader.numDocs(), equalTo(flushFirst ? numDocs : 0));
         }
         commit.close();
+        // Make the global checkpoint in sync with the local checkpoint.
+        if (isPrimary) {
+            final String allocationId = shard.shardRouting.allocationId().getId();
+            shard.updateLocalCheckpointForShard(allocationId, numDocs + moreDocs - 1);
+            shard.updateGlobalCheckpointForShard(allocationId, shard.getLocalCheckpoint());
+        } else {
+            shard.updateGlobalCheckpointOnReplica(numDocs + moreDocs - 1, "test");
+        }
         flushShard(shard, true);
 
         // check it's clean up
diff --git a/core/src/test/java/org/elasticsearch/index/translog/SnapshotMatchers.java b/core/src/test/java/org/elasticsearch/index/translog/SnapshotMatchers.java
index 4ca6057bd6b..21f7dd9481c 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/SnapshotMatchers.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/SnapshotMatchers.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.index.translog;
 
+import com.carrotsearch.hppc.LongHashSet;
+import com.carrotsearch.hppc.LongSet;
 import org.elasticsearch.ElasticsearchException;
 import org.hamcrest.Description;
 import org.hamcrest.Matcher;
@@ -61,6 +63,13 @@ public final class SnapshotMatchers {
         return new ContainingInAnyOrderMatcher(expectedOperations);
     }
 
+    /**
+     * Consumes a snapshot and makes sure that its operations have all seqno between minSeqNo(inclusive) and maxSeqNo(inclusive).
+     */
+    public static Matcher containsSeqNoRange(long minSeqNo, long maxSeqNo) {
+        return new ContainingSeqNoRangeMatcher(minSeqNo, maxSeqNo);
+    }
+
     public static class SizeMatcher extends TypeSafeMatcher {
 
         private final int size;
@@ -190,4 +199,45 @@ public final class SnapshotMatchers {
                 .appendText(" in any order.");
         }
     }
+
+    static class ContainingSeqNoRangeMatcher extends TypeSafeMatcher {
+        private final long minSeqNo;
+        private final long maxSeqNo;
+        private final List notFoundSeqNo = new ArrayList<>();
+
+        ContainingSeqNoRangeMatcher(long minSeqNo, long maxSeqNo) {
+            this.minSeqNo = minSeqNo;
+            this.maxSeqNo = maxSeqNo;
+        }
+
+        @Override
+        protected boolean matchesSafely(Translog.Snapshot snapshot) {
+            try {
+                final LongSet seqNoList = new LongHashSet();
+                Translog.Operation op;
+                while ((op = snapshot.next()) != null) {
+                    seqNoList.add(op.seqNo());
+                }
+                for (long i = minSeqNo; i <= maxSeqNo; i++) {
+                    if (seqNoList.contains(i) == false) {
+                        notFoundSeqNo.add(i);
+                    }
+                }
+                return notFoundSeqNo.isEmpty();
+            } catch (IOException ex) {
+                throw new ElasticsearchException("failed to read snapshot content", ex);
+            }
+        }
+
+        @Override
+        protected void describeMismatchSafely(Translog.Snapshot snapshot, Description mismatchDescription) {
+            mismatchDescription
+                .appendText("not found seqno ").appendValueList("[", ", ", "]", notFoundSeqNo);
+        }
+
+        @Override
+        public void describeTo(Description description) {
+            description.appendText("snapshot contains all seqno from [" + minSeqNo + " to " + maxSeqNo + "]");
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
index f62d292730e..39fc182623f 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
@@ -53,6 +53,7 @@ public class TranslogDeletionPolicyTests extends ESTestCase {
             assertMinGenRequired(deletionPolicy, readersAndWriter, 1L);
             final int committedReader = randomIntBetween(0, allGens.size() - 1);
             final long committedGen = allGens.get(committedReader).generation;
+            deletionPolicy.setTranslogGenerationOfLastCommit(randomLongBetween(committedGen, Long.MAX_VALUE));
             deletionPolicy.setMinTranslogGenerationForRecovery(committedGen);
             assertMinGenRequired(deletionPolicy, readersAndWriter, committedGen);
         } finally {
@@ -109,6 +110,7 @@ public class TranslogDeletionPolicyTests extends ESTestCase {
         allGens.add(readersAndWriter.v2());
         try {
             TranslogDeletionPolicy deletionPolicy = new MockDeletionPolicy(now, Long.MAX_VALUE, Long.MAX_VALUE);
+            deletionPolicy.setTranslogGenerationOfLastCommit(Long.MAX_VALUE);
             deletionPolicy.setMinTranslogGenerationForRecovery(Long.MAX_VALUE);
             int selectedReader = randomIntBetween(0, allGens.size() - 1);
             final long selectedGenerationByAge = allGens.get(selectedReader).generation;
@@ -122,6 +124,7 @@ public class TranslogDeletionPolicyTests extends ESTestCase {
             // make a new policy as committed gen can't go backwards (for now)
             deletionPolicy = new MockDeletionPolicy(now, size, maxAge);
             long committedGen = randomFrom(allGens).generation;
+            deletionPolicy.setTranslogGenerationOfLastCommit(randomLongBetween(committedGen, Long.MAX_VALUE));
             deletionPolicy.setMinTranslogGenerationForRecovery(committedGen);
             assertMinGenRequired(deletionPolicy, readersAndWriter,
                 Math.min(committedGen, Math.max(selectedGenerationByAge, selectedGenerationBySize)));
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index 593e1059215..dc050949fe3 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -153,17 +153,20 @@ public class TranslogTests extends ESTestCase {
     }
 
     private void markCurrentGenAsCommitted(Translog translog) throws IOException {
-        commit(translog, translog.currentFileGeneration());
+        long genToCommit = translog.currentFileGeneration();
+        long genToRetain = randomLongBetween(translog.getDeletionPolicy().getMinTranslogGenerationForRecovery(), genToCommit);
+        commit(translog, genToRetain, genToCommit);
     }
 
     private void rollAndCommit(Translog translog) throws IOException {
         translog.rollGeneration();
-        commit(translog, translog.currentFileGeneration());
+        markCurrentGenAsCommitted(translog);
     }
 
-    private void commit(Translog translog, long genToCommit) throws IOException {
+    private void commit(Translog translog, long genToRetain, long genToCommit) throws IOException {
         final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy();
-        deletionPolicy.setMinTranslogGenerationForRecovery(genToCommit);
+        deletionPolicy.setTranslogGenerationOfLastCommit(genToCommit);
+        deletionPolicy.setMinTranslogGenerationForRecovery(genToRetain);
         long minGenRequired = deletionPolicy.minTranslogGenRequired(translog.getReaders(), translog.getCurrent());
         translog.trimUnreferencedReaders();
         assertThat(minGenRequired, equalTo(translog.getMinFileGeneration()));
@@ -440,6 +443,31 @@ public class TranslogTests extends ESTestCase {
         }
     }
 
+    public void testUncommittedOperations() throws Exception {
+        final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy();
+        deletionPolicy.setRetentionAgeInMillis(randomLong());
+        deletionPolicy.setRetentionSizeInBytes(randomLong());
+
+        final int operations = scaledRandomIntBetween(10, 100);
+        int uncommittedOps = 0;
+        int operationsInLastGen = 0;
+        for (int i = 0; i < operations; i++) {
+            translog.add(new Translog.Index("test", Integer.toString(i), i, new byte[]{1}));
+            uncommittedOps++;
+            operationsInLastGen++;
+            if (rarely()) {
+                translog.rollGeneration();
+                operationsInLastGen = 0;
+            }
+            assertThat(translog.uncommittedOperations(), equalTo(uncommittedOps));
+            if (frequently()) {
+                markCurrentGenAsCommitted(translog);
+                assertThat(translog.uncommittedOperations(), equalTo(operationsInLastGen));
+                uncommittedOps = operationsInLastGen;
+            }
+        }
+    }
+
     public void testTotalTests() {
         final TranslogStats total = new TranslogStats();
         final int n = randomIntBetween(0, 16);
@@ -824,6 +852,7 @@ public class TranslogTests extends ESTestCase {
                                 translog.rollGeneration();
                                 // expose the new checkpoint (simulating a commit), before we trim the translog
                                 lastCommittedLocalCheckpoint.set(localCheckpoint);
+                                deletionPolicy.setTranslogGenerationOfLastCommit(translog.currentFileGeneration());
                                 deletionPolicy.setMinTranslogGenerationForRecovery(
                                     translog.getMinGenerationForSeqNo(localCheckpoint + 1).translogFileGeneration);
                                 translog.trimUnreferencedReaders();
@@ -1822,6 +1851,7 @@ public class TranslogTests extends ESTestCase {
         translog.close();
         TranslogConfig config = translog.getConfig();
         final TranslogDeletionPolicy deletionPolicy = new TranslogDeletionPolicy(-1, -1);
+        deletionPolicy.setTranslogGenerationOfLastCommit(randomLongBetween(comittedGeneration, Long.MAX_VALUE));
         deletionPolicy.setMinTranslogGenerationForRecovery(comittedGeneration);
         translog = new Translog(config, translog.getTranslogUUID(), deletionPolicy, () -> SequenceNumbers.UNASSIGNED_SEQ_NO);
         assertThat(translog.getMinFileGeneration(), equalTo(1L));
@@ -1867,6 +1897,7 @@ public class TranslogTests extends ESTestCase {
                     translog.rollGeneration();
                 }
             }
+            deletionPolicy.setTranslogGenerationOfLastCommit(randomLongBetween(comittedGeneration, translog.currentFileGeneration()));
             deletionPolicy.setMinTranslogGenerationForRecovery(comittedGeneration);
             fail.failRandomly();
             try {
@@ -1876,6 +1907,7 @@ public class TranslogTests extends ESTestCase {
             }
         }
         final TranslogDeletionPolicy deletionPolicy = new TranslogDeletionPolicy(-1, -1);
+        deletionPolicy.setTranslogGenerationOfLastCommit(randomLongBetween(comittedGeneration, Long.MAX_VALUE));
         deletionPolicy.setMinTranslogGenerationForRecovery(comittedGeneration);
         try (Translog translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.UNASSIGNED_SEQ_NO)) {
             // we don't know when things broke exactly
@@ -2413,8 +2445,9 @@ public class TranslogTests extends ESTestCase {
         for (int i = 0; i <= rolls; i++) {
             assertFileIsPresent(translog, generation + i);
         }
-        commit(translog, generation + rolls);
-        assertThat(translog.currentFileGeneration(), equalTo(generation + rolls ));
+        long minGenForRecovery = randomLongBetween(generation, generation + rolls);
+        commit(translog, minGenForRecovery, generation + rolls);
+        assertThat(translog.currentFileGeneration(), equalTo(generation + rolls));
         assertThat(translog.uncommittedOperations(), equalTo(0));
         if (longRetention) {
             for (int i = 0; i <= rolls; i++) {
@@ -2423,17 +2456,19 @@ public class TranslogTests extends ESTestCase {
             deletionPolicy.setRetentionAgeInMillis(randomBoolean() ? 100 : -1);
             assertBusy(() -> {
                 translog.trimUnreferencedReaders();
-                for (int i = 0; i < rolls; i++) {
-                    assertFileDeleted(translog, generation + i);
+                for (long i = 0; i < minGenForRecovery; i++) {
+                    assertFileDeleted(translog, i);
                 }
             });
         } else {
             // immediate cleanup
-            for (int i = 0; i < rolls; i++) {
-                assertFileDeleted(translog, generation + i);
+            for (long i = 0; i < minGenForRecovery; i++) {
+                assertFileDeleted(translog, i);
             }
         }
-        assertFileIsPresent(translog, generation + rolls);
+        for (long i = minGenForRecovery; i < generation + rolls; i++) {
+            assertFileIsPresent(translog, i);
+        }
     }
 
     public void testMinSeqNoBasedAPI() throws IOException {
@@ -2516,10 +2551,8 @@ public class TranslogTests extends ESTestCase {
                 translog.rollGeneration();
             }
         }
-
-        final long generation =
-                randomIntBetween(1, Math.toIntExact(translog.currentFileGeneration()));
-        commit(translog, generation);
+        long lastGen = randomLongBetween(1, translog.currentFileGeneration());
+        commit(translog, randomLongBetween(1, lastGen), lastGen);
     }
 
     public void testAcquiredLockIsPassedToDeletionPolicy() throws IOException {
@@ -2531,7 +2564,9 @@ public class TranslogTests extends ESTestCase {
                 translog.rollGeneration();
             }
             if (rarely()) {
-                commit(translog, randomLongBetween(deletionPolicy.getMinTranslogGenerationForRecovery(), translog.currentFileGeneration()));
+                long lastGen = randomLongBetween(deletionPolicy.getTranslogGenerationOfLastCommit(), translog.currentFileGeneration());
+                long minGen = randomLongBetween(deletionPolicy.getMinTranslogGenerationForRecovery(), lastGen);
+                commit(translog, minGen, lastGen);
             }
             if (frequently()) {
                 long minGen;
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
index 55b7e22eb8a..689627d6642 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.indices.recovery;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.NoMergePolicy;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.UUIDs;
@@ -98,20 +99,41 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase {
     }
 
     public void testRecoveryWithOutOfOrderDelete() throws Exception {
+        /*
+         * The flow of this test:
+         * - delete #1
+         * - roll generation (to create gen 2)
+         * - index #0
+         * - index #3
+         * - flush (commit point has max_seqno 3, and local checkpoint 1 -> points at gen 2, previous commit point is maintained)
+         * - index #2
+         * - index #5
+         * - If flush and the translog retention disabled, delete #1 will be removed while index #0 is still retained and replayed.
+         */
         try (ReplicationGroup shards = createGroup(1)) {
             shards.startAll();
             // create out of order delete and index op on replica
             final IndexShard orgReplica = shards.getReplicas().get(0);
+            final String indexName = orgReplica.shardId().getIndexName();
+
+            // delete #1
             orgReplica.applyDeleteOperationOnReplica(1, 2, "type", "id", VersionType.EXTERNAL, u -> {});
             orgReplica.getTranslog().rollGeneration(); // isolate the delete in it's own generation
+            // index #0
             orgReplica.applyIndexOperationOnReplica(0, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
-                SourceToParse.source(orgReplica.shardId().getIndexName(), "type", "id", new BytesArray("{}"), XContentType.JSON),
-                u -> {});
-
-            // index a second item into the second generation, skipping seq# 2. Local checkpoint is now 1, which will make this generation
-            // stick around
+                SourceToParse.source(indexName, "type", "id", new BytesArray("{}"), XContentType.JSON), u -> {});
+            // index #3
             orgReplica.applyIndexOperationOnReplica(3, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
-                SourceToParse.source(orgReplica.shardId().getIndexName(), "type", "id2", new BytesArray("{}"), XContentType.JSON), u -> {});
+                SourceToParse.source(indexName, "type", "id-3", new BytesArray("{}"), XContentType.JSON), u -> {});
+            // Flushing a new commit with local checkpoint=1 allows to delete the translog gen #1.
+            orgReplica.flush(new FlushRequest().force(true).waitIfOngoing(true));
+            // index #2
+            orgReplica.applyIndexOperationOnReplica(2, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+                SourceToParse.source(indexName, "type", "id-2", new BytesArray("{}"), XContentType.JSON), u -> {});
+            orgReplica.updateGlobalCheckpointOnReplica(3L, "test");
+            // index #5 -> force NoOp #4.
+            orgReplica.applyIndexOperationOnReplica(5, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
+                SourceToParse.source(indexName, "type", "id-5", new BytesArray("{}"), XContentType.JSON), u -> {});
 
             final int translogOps;
             if (randomBoolean()) {
@@ -120,18 +142,17 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase {
                     IndexMetaData.Builder builder = IndexMetaData.builder(orgReplica.indexSettings().getIndexMetaData());
                     builder.settings(Settings.builder().put(orgReplica.indexSettings().getSettings())
                         .put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), "-1")
-                        .put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), "-1")
-                    );
+                        .put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), "-1"));
                     orgReplica.indexSettings().updateIndexMetaData(builder.build());
                     orgReplica.onSettingsChanged();
-                    translogOps = 3; // 2 ops + seqno gaps
+                    translogOps = 5; // 4 ops + seqno gaps (delete #1 is removed but index #0 will be replayed).
                 } else {
                     logger.info("--> flushing shard (translog will be retained)");
-                    translogOps = 4; // 3 ops + seqno gaps
+                    translogOps = 6; // 5 ops + seqno gaps
                 }
                 flushShard(orgReplica);
             } else {
-                translogOps = 4; // 3 ops + seqno gaps
+                translogOps = 6; // 5 ops + seqno gaps
             }
 
             final IndexShard orgPrimary = shards.getPrimary();
@@ -139,7 +160,7 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase {
 
             IndexShard newReplica = shards.addReplicaWithExistingPath(orgPrimary.shardPath(), orgPrimary.routingEntry().currentNodeId());
             shards.recoverReplica(newReplica);
-            shards.assertAllEqual(1);
+            shards.assertAllEqual(3);
 
             assertThat(newReplica.getTranslog().totalOperations(), equalTo(translogOps));
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 01ad6763a8c..84e750f6e28 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -461,6 +461,13 @@ public abstract class ESTestCase extends LuceneTestCase {
         return RandomNumbers.randomIntBetween(random(), min, max);
     }
 
+    /**
+     * A random long number between min (inclusive) and max (inclusive).
+     */
+    public static long randomLongBetween(long min, long max) {
+        return RandomNumbers.randomLongBetween(random(), min, max);
+    }
+
     /**
      * Returns a "scaled" number of iterations for loops which can have a variable
      * iteration count. This method is effectively

From 55738ac1b957456022bc023ce7cf11ca3319b2e6 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen 
Date: Tue, 12 Dec 2017 20:58:50 -0500
Subject: [PATCH 253/297] TEST: Update translog gen of the last commit

The test testWithRandomException was not updated accordingly to the
latest translog policy. Method setTranslogGenerationOfLastCommit should
be called before whenever setMinTranslogGenerationForRecovery is called.

Relates #27606
---
 .../java/org/elasticsearch/index/translog/TranslogTests.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index dc050949fe3..e8229569ac4 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -108,7 +108,6 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 import java.util.stream.LongStream;
 
-import static com.carrotsearch.randomizedtesting.RandomizedTest.randomLongBetween;
 import static org.elasticsearch.common.util.BigArrays.NON_RECYCLING_INSTANCE;
 import static org.elasticsearch.index.translog.SnapshotMatchers.containsOperationsInAnyOrder;
 import static org.elasticsearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy;
@@ -2224,6 +2223,7 @@ public class TranslogTests extends ESTestCase {
                             unsynced.clear();
                             failableTLog.rollGeneration();
                             committing = true;
+                            failableTLog.getDeletionPolicy().setTranslogGenerationOfLastCommit(failableTLog.currentFileGeneration());
                             failableTLog.getDeletionPolicy().setMinTranslogGenerationForRecovery(failableTLog.currentFileGeneration());
                             failableTLog.trimUnreferencedReaders();
                             committing = false;
@@ -2266,6 +2266,7 @@ public class TranslogTests extends ESTestCase {
             if (randomBoolean()) {
                 try {
                     TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy();
+                    deletionPolicy.setTranslogGenerationOfLastCommit(minGenForRecovery);
                     deletionPolicy.setMinTranslogGenerationForRecovery(minGenForRecovery);
                     IOUtils.close(getFailableTranslog(fail, config, randomBoolean(), false, generationUUID, deletionPolicy));
                 } catch (TranslogException | MockDirectoryWrapper.FakeIOException ex) {
@@ -2277,6 +2278,7 @@ public class TranslogTests extends ESTestCase {
 
             fail.failNever(); // we don't wanna fail here but we might since we write a new checkpoint and create a new tlog file
             TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy();
+            deletionPolicy.setTranslogGenerationOfLastCommit(minGenForRecovery);
             deletionPolicy.setMinTranslogGenerationForRecovery(minGenForRecovery);
             try (Translog translog = new Translog(config, generationUUID, deletionPolicy, () -> SequenceNumbers.UNASSIGNED_SEQ_NO);
                  Translog.Snapshot snapshot = translog.newSnapshotFromGen(minGenForRecovery)) {

From 247efa86bff672aa4cca0c74e08ba26e10ab06d2 Mon Sep 17 00:00:00 2001
From: Boaz Leskes 
Date: Wed, 13 Dec 2017 14:52:05 +0100
Subject: [PATCH 254/297] remove stale comment in IndexShard

---
 .../src/main/java/org/elasticsearch/index/shard/IndexShard.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index b3847cd1b4d..61ec22a41a8 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -141,7 +141,6 @@ import java.io.PrintStream;
 import java.nio.channels.ClosedByInterruptException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
@@ -415,7 +414,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                 }
 
                 if (newRouting.primary() && currentRouting.isRelocationTarget() == false) {
-                    // there was no primary context hand-off in < 6.0.0, need to manually activate the shard
                     getEngine().seqNoService().activatePrimaryMode(getEngine().seqNoService().getLocalCheckpoint());
                 }
 

From b69923f1124fca4c0f8a4ba8e55fc5c324c03bb7 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux 
Date: Wed, 13 Dec 2017 16:45:55 +0100
Subject: [PATCH 255/297] Remove some unused code (#27792)

This commit removes some unused code.
---
 .../FailureTrackingResponseListenerTests.java    |  4 ++++
 .../elasticsearch/action/bulk/BulkProcessor.java |  2 --
 .../action/get/TransportGetAction.java           |  2 --
 .../org/elasticsearch/bootstrap/Bootstrap.java   |  7 +++----
 .../cluster/routing/IndexShardRoutingTable.java  | 13 -------------
 .../cluster/routing/RoutingNodes.java            |  2 +-
 .../common/inject/spi/Elements.java              | 13 -------------
 .../org/elasticsearch/common/util/BigArrays.java |  1 -
 .../util/concurrent/ResizableBlockingQueue.java  |  7 -------
 .../org/elasticsearch/env/NodeEnvironment.java   |  7 -------
 .../index/query/ScriptQueryBuilder.java          |  2 --
 .../elasticsearch/index/shard/IndexShard.java    |  2 +-
 .../index/shard/IndexShardOperationPermits.java  |  5 +----
 .../org/elasticsearch/index/shard/ShardPath.java |  9 ---------
 .../blobstore/BlobStoreRepository.java           |  2 +-
 .../rest/action/cat/RestShardsAction.java        |  1 -
 .../phrase/DirectCandidateGeneratorBuilder.java  |  5 -----
 .../suggest/term/TermSuggestionBuilder.java      |  4 ----
 .../shard/IndexShardOperationPermitsTests.java   |  2 +-
 .../discovery/ec2/Ec2DiscoveryPlugin.java        |  7 ++-----
 .../discovery/file/FileBasedDiscoveryPlugin.java |  2 --
 .../file/FileBasedUnicastHostsProvider.java      |  4 ----
 .../plugin/discovery/gce/GceDiscoveryPlugin.java |  2 --
 .../repositories/s3/InternalAwsS3Service.java    | 16 ++++------------
 .../s3/AbstractS3SnapshotRestoreTest.java        |  2 --
 25 files changed, 18 insertions(+), 105 deletions(-)

diff --git a/client/rest/src/test/java/org/elasticsearch/client/FailureTrackingResponseListenerTests.java b/client/rest/src/test/java/org/elasticsearch/client/FailureTrackingResponseListenerTests.java
index f6ec388d09d..3bbb2acbae2 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/FailureTrackingResponseListenerTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/FailureTrackingResponseListenerTests.java
@@ -41,6 +41,10 @@ public class FailureTrackingResponseListenerTests extends RestClientTestCase {
         MockResponseListener responseListener = new MockResponseListener();
         RestClient.FailureTrackingResponseListener listener = new RestClient.FailureTrackingResponseListener(responseListener);
 
+        final Response response = mockResponse();
+        listener.onSuccess(response);
+        assertSame(response, responseListener.response.get());
+        assertNull(responseListener.exception.get());
     }
 
     public void testOnFailure() {
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
index 372837521dd..668dd230f60 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
@@ -185,7 +185,6 @@ public class BulkProcessor implements Closeable {
 
     private BulkRequest bulkRequest;
     private final BulkRequestHandler bulkRequestHandler;
-    private final Scheduler scheduler;
     private final Runnable onClose;
 
     private volatile boolean closed = false;
@@ -196,7 +195,6 @@ public class BulkProcessor implements Closeable {
         this.bulkActions = bulkActions;
         this.bulkSize = bulkSize.getBytes();
         this.bulkRequest = new BulkRequest();
-        this.scheduler = scheduler;
         this.bulkRequestHandler = new BulkRequestHandler(consumer, backoffPolicy, listener, scheduler, concurrentRequests);
         // Start period flushing task after everything is setup
         this.cancellableFlushTask = startFlushTask(flushInterval, scheduler);
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
index d14db67744d..599a6655e02 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -69,7 +68,6 @@ public class TransportGetAction extends TransportSingleShardAction {
 
         // Retrieve which nodes we can potentially send the query to
         final Set nodeIds = getAllNodeIds(shards);
-        final int nodeCount = nodeIds.size();
-
         final Map> nodeStats = getNodeStats(nodeIds, collector);
 
         // Retrieve all the nodes the shards exist on
@@ -424,16 +421,6 @@ public class IndexShardRoutingTable implements Iterable {
         }
     }
 
-    /**
-     * Returns true if no primaries are active or initializing for this shard
-     */
-    private boolean noPrimariesActive() {
-        if (!primaryAsList.isEmpty() && !primaryAsList.get(0).active() && !primaryAsList.get(0).initializing()) {
-            return true;
-        }
-        return false;
-    }
-
     /**
      * Returns an iterator only on the primary shard.
      */
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index 43711525bcf..aaf34d1e180 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -632,7 +632,7 @@ public class RoutingNodes implements Iterable {
         // if the activeReplica was relocating before this call to failShard, its relocation was cancelled earlier when we
         // failed initializing replica shards (and moved replica relocation source back to started)
         assert activeReplica.started() : "replica relocation should have been cancelled: " + activeReplica;
-        ShardRouting primarySwappedCandidate = promoteActiveReplicaShardToPrimary(activeReplica);
+        promoteActiveReplicaShardToPrimary(activeReplica);
         routingChangesObserver.replicaPromoted(activeReplica);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java b/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java
index bfb084dd478..6d930953e29 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java
@@ -19,7 +19,6 @@ package org.elasticsearch.common.inject.spi;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.inject.Binding;
 import org.elasticsearch.common.inject.Key;
 import org.elasticsearch.common.inject.MembersInjector;
 import org.elasticsearch.common.inject.Module;
@@ -60,18 +59,6 @@ import java.util.Set;
  * @since 2.0
  */
 public final class Elements {
-    private static final BindingTargetVisitor GET_INSTANCE_VISITOR
-            = new DefaultBindingTargetVisitor() {
-        @Override
-        public Object visit(InstanceBinding binding) {
-            return binding.getInstance();
-        }
-
-        @Override
-        protected Object visitOther(Binding binding) {
-            throw new IllegalArgumentException();
-        }
-    };
 
     /**
      * Records the elements executed by {@code modules}.
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
index ec46a2b937f..2d3fde01964 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
@@ -749,7 +749,6 @@ public class BigArrays implements Releasable {
      * @param size          the initial length of the array
      */
     public  ObjectArray newObjectArray(long size) {
-        final ObjectArray array;
         if (size > OBJECT_PAGE_SIZE) {
             // when allocating big arrays, we want to first ensure we have the capacity by
             // checking with the circuit breaker before attempting to allocate
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/ResizableBlockingQueue.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/ResizableBlockingQueue.java
index ca6f6030bb0..57240b6e1cf 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/ResizableBlockingQueue.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/ResizableBlockingQueue.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.util.concurrent;
 
 import java.util.concurrent.BlockingQueue;
-import org.elasticsearch.common.SuppressForbidden;
 
 /**
  * Extends the {@code SizeBlockingQueue} to add the {@code adjustCapacity} method, which will adjust
@@ -35,12 +34,6 @@ final class ResizableBlockingQueue extends SizeBlockingQueue {
         this.capacity = initialCapacity;
     }
 
-    @SuppressForbidden(reason = "optimalCapacity is non-negative, therefore the difference cannot be < -Integer.MAX_VALUE")
-    private int getChangeAmount(int optimalCapacity) {
-        assert optimalCapacity >= 0 : "optimal capacity should always be positive, got: " + optimalCapacity;
-        return Math.abs(optimalCapacity - this.capacity);
-    }
-
     @Override
     public int capacity() {
         return this.capacity;
diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index 9c583351174..172e3687e39 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -278,13 +278,6 @@ public final class NodeEnvironment  implements Closeable {
         return path.resolve(NODES_FOLDER).resolve(Integer.toString(nodeLockId));
     }
 
-    /** Returns true if the directory is empty */
-    private static boolean dirEmpty(final Path path) throws IOException {
-        try (DirectoryStream stream = Files.newDirectoryStream(path)) {
-            return stream.iterator().hasNext() == false;
-        }
-    }
-
     private static void releaseAndNullLocks(Lock[] locks) {
         for (int i = 0; i < locks.length; i++) {
             if (locks[i] != null) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
index 6e0b8159ded..3d217ab36a2 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
@@ -44,8 +44,6 @@ import java.util.Objects;
 public class ScriptQueryBuilder extends AbstractQueryBuilder {
     public static final String NAME = "script";
 
-    private static final ParseField PARAMS_FIELD = new ParseField("params");
-
     private final Script script;
 
     public ScriptQueryBuilder(Script script) {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 61ec22a41a8..444f3f56d9a 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -303,7 +303,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         } else {
             cachingPolicy = new UsageTrackingQueryCachingPolicy();
         }
-        indexShardOperationPermits = new IndexShardOperationPermits(shardId, logger, threadPool);
+        indexShardOperationPermits = new IndexShardOperationPermits(shardId, threadPool);
         searcherWrapper = indexSearcherWrapper;
         primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id());
         refreshListeners = buildRefreshListeners();
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java
index 3f6d443aa80..75645198f5b 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java
@@ -49,7 +49,6 @@ import java.util.function.Supplier;
 final class IndexShardOperationPermits implements Closeable {
 
     private final ShardId shardId;
-    private final Logger logger;
     private final ThreadPool threadPool;
 
     static final int TOTAL_PERMITS = Integer.MAX_VALUE;
@@ -62,12 +61,10 @@ final class IndexShardOperationPermits implements Closeable {
      * Construct operation permits for the specified shards.
      *
      * @param shardId    the shard
-     * @param logger     the logger for the shard
      * @param threadPool the thread pool (used to execute delayed operations)
      */
-    IndexShardOperationPermits(final ShardId shardId, final Logger logger, final ThreadPool threadPool) {
+    IndexShardOperationPermits(final ShardId shardId, final ThreadPool threadPool) {
         this.shardId = shardId;
-        this.logger = logger;
         this.threadPool = threadPool;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
index 4a18bcdd5c7..8ff651e8af3 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
@@ -33,9 +33,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 public final class ShardPath {
     public static final String INDEX_FOLDER_NAME = "index";
@@ -198,13 +196,6 @@ public final class ShardPath {
             NodeEnvironment.NodePath bestPath = getPathWithMostFreeSpace(env);
 
             if (paths.length != 1) {
-                int shardCount = indexSettings.getNumberOfShards();
-                // Maximum number of shards that a path should have for a particular index assuming
-                // all the shards were assigned to this node. For example, with a node with 4 data
-                // paths and an index with 9 primary shards, the maximum number of shards per path
-                // would be 3.
-                int maxShardsPerPath = Math.floorDiv(shardCount, paths.length) + ((shardCount % paths.length) == 0 ? 0 : 1);
-
                 Map pathToShardCount = env.shardCountPerPath(shardId.getIndex());
 
                 // Compute how much space there is on each path
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 9afbb528782..06812be5aab 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -1499,7 +1499,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                         }
                     }
                 }
-                final RecoveryState.Index index = recoveryState.getIndex();
+
                 if (filesToRecover.isEmpty()) {
                     logger.trace("no files to recover, all exists within the local store");
                 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index d985c86d9b3..a5a88006161 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -194,7 +194,6 @@ public class RestShardsAction extends AbstractCatAction {
             table.addCell(shard.getIndexName());
             table.addCell(shard.id());
 
-            IndexMetaData indexMeta = state.getState().getMetaData().getIndexSafe(shard.index());
             if (shard.primary()) {
                 table.addCell("p");
             } else {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
index 5e0ed7749f7..bb824d89c0e 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
@@ -31,8 +31,6 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -47,9 +45,6 @@ import java.util.function.Consumer;
 
 public final class DirectCandidateGeneratorBuilder implements CandidateGenerator {
 
-    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(
-            Loggers.getLogger(DirectCandidateGeneratorBuilder.class));
-
     private static final String TYPE = "direct_generator";
 
     public static final ParseField DIRECT_GENERATOR_FIELD = new ParseField(TYPE);
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
index 1833bb5a90e..b0d9891087f 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
@@ -30,8 +30,6 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -69,8 +67,6 @@ import static org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBu
  */
 public class TermSuggestionBuilder extends SuggestionBuilder {
 
-    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(TermSuggestionBuilder.class));
-
     private static final String SUGGESTION_NAME = "term";
 
     private SuggestMode suggestMode = SuggestMode.MISSING;
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java
index 0f6c8fbee02..ab5192dfc3e 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java
@@ -85,7 +85,7 @@ public class IndexShardOperationPermitsTests extends ESTestCase {
 
     @Before
     public void createIndexShardOperationsLock() {
-         permits = new IndexShardOperationPermits(new ShardId("blubb", "id", 0), logger, threadPool);
+        permits = new IndexShardOperationPermits(new ShardId("blubb", "id", 0), threadPool);
     }
 
     @After
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java
index 50c5dac4b75..9669bb74769 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java
@@ -25,7 +25,6 @@ import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
@@ -36,12 +35,12 @@ import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.UncheckedIOException;
 import java.io.BufferedReader;
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.io.Closeable;
+import java.io.UncheckedIOException;
 import java.net.URL;
 import java.net.URLConnection;
 import java.nio.charset.StandardCharsets;
@@ -56,8 +55,6 @@ import java.util.function.Supplier;
 public class Ec2DiscoveryPlugin extends Plugin implements DiscoveryPlugin, Closeable {
 
     private static Logger logger = Loggers.getLogger(Ec2DiscoveryPlugin.class);
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
-
     public static final String EC2 = "ec2";
 
     static {
diff --git a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java
index a8f3337d50d..fb37b3bc011 100644
--- a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java
+++ b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java
@@ -23,7 +23,6 @@ import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -59,7 +58,6 @@ import java.util.function.Supplier;
 public class FileBasedDiscoveryPlugin extends Plugin implements DiscoveryPlugin {
 
     private static final Logger logger = Loggers.getLogger(FileBasedDiscoveryPlugin.class);
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
     private final Settings settings;
     private final Path configPath;
diff --git a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProvider.java b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProvider.java
index ee5f6c08b91..1029f907a66 100644
--- a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProvider.java
+++ b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProvider.java
@@ -23,7 +23,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.env.Environment;
@@ -38,7 +37,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -67,8 +65,6 @@ class FileBasedUnicastHostsProvider extends AbstractComponent implements Unicast
 
     private final Path unicastHostsFilePath;
 
-    private final AtomicLong nodeIdGenerator = new AtomicLong(); // generates unique ids for the node
-
     private final TimeValue resolveTimeout;
 
     FileBasedUnicastHostsProvider(Environment environment, TransportService transportService, ExecutorService executorService) {
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
index f636dcaba0c..139416c1e64 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
@@ -29,7 +29,6 @@ import org.elasticsearch.cloud.gce.GceInstancesServiceImpl;
 import org.elasticsearch.cloud.gce.GceMetadataService;
 import org.elasticsearch.cloud.gce.network.GceNameResolver;
 import org.elasticsearch.cloud.gce.util.Access;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
@@ -53,7 +52,6 @@ public class GceDiscoveryPlugin extends Plugin implements DiscoveryPlugin, Close
     public static final String GCE = "gce";
     private final Settings settings;
     private static final Logger logger = Loggers.getLogger(GceDiscoveryPlugin.class);
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
     // stashed when created in order to properly close
     private final SetOnce gceInstancesService = new SetOnce<>();
 
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java
index 58a130bcd95..2b21a81d91f 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java
@@ -19,10 +19,6 @@
 
 package org.elasticsearch.repositories.s3;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.function.Function;
-
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
@@ -41,6 +37,10 @@ import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+
 
 class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service {
 
@@ -139,14 +139,6 @@ class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Se
         }
     }
 
-    /** Returns the value for a given setting from the repository, or returns the fallback value. */
-    private static  T getRepoValue(Settings repositorySettings, Setting repositorySetting, T fallback) {
-        if (repositorySetting.exists(repositorySettings)) {
-            return repositorySetting.get(repositorySettings);
-        }
-        return fallback;
-    }
-
     @Override
     protected void doStart() throws ElasticsearchException {
     }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
index 6f4eff93c23..b40dc75c837 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
@@ -31,7 +31,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotR
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ClusterAdminClient;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.repositories.RepositoryVerificationException;
@@ -181,7 +180,6 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
 
         Settings settings = internalCluster().getInstance(Settings.class);
         Settings bucket = settings.getByPrefix("repositories.s3.");
-        RepositoryMetaData metadata = new RepositoryMetaData("test-repo", "fs", Settings.EMPTY);
         AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(repositorySettings);
 
         String bucketName = bucket.get("bucket");

From 28f65123195c41e51e957ca0df8c68519418ee91 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux 
Date: Wed, 13 Dec 2017 16:47:01 +0100
Subject: [PATCH 256/297] [Test] Fix MigrationDocumentationIT.testClusterHealth
 (#27774)

Closes #27754
---
 .../MigrationDocumentationIT.java             | 26 +++++++++++--------
 docs/java-rest/high-level/migration.asciidoc  |  6 ++---
 2 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java
index 21fa1152ccd..1f57ceb4bfb 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java
@@ -26,6 +26,7 @@ import org.apache.http.nio.entity.NStringEntity;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
@@ -43,6 +44,7 @@ import java.io.InputStream;
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 
@@ -67,7 +69,7 @@ public class MigrationDocumentationIT extends ESRestHighLevelClientTestCase {
     public void testCreateIndex() throws IOException {
         RestHighLevelClient client = highLevelClient();
         {
-            //tag::migration-create-inded
+            //tag::migration-create-index
             Settings indexSettings = Settings.builder() // <1>
                     .put(SETTING_NUMBER_OF_SHARDS, 1)
                     .put(SETTING_NUMBER_OF_REPLICAS, 0)
@@ -95,7 +97,7 @@ public class MigrationDocumentationIT extends ESRestHighLevelClientTestCase {
             if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
                 // <7>
             }
-            //end::migration-create-inded
+            //end::migration-create-index
             assertEquals(200, response.getStatusLine().getStatusCode());
         }
     }
@@ -104,7 +106,8 @@ public class MigrationDocumentationIT extends ESRestHighLevelClientTestCase {
         RestHighLevelClient client = highLevelClient();
         {
             //tag::migration-cluster-health
-            Response response = client.getLowLevelClient().performRequest("GET", "/_cluster/health"); // <1>
+            Map parameters = singletonMap("wait_for_status", "green");
+            Response response = client.getLowLevelClient().performRequest("GET", "/_cluster/health", parameters); // <1>
 
             ClusterHealthStatus healthStatus;
             try (InputStream is = response.getEntity().getContent()) { // <2>
@@ -120,7 +123,7 @@ public class MigrationDocumentationIT extends ESRestHighLevelClientTestCase {
         }
     }
 
-    public void testRequests() throws IOException {
+    public void testRequests() throws Exception {
         RestHighLevelClient client = highLevelClient();
         {
             //tag::migration-request-ctor
@@ -133,13 +136,6 @@ public class MigrationDocumentationIT extends ESRestHighLevelClientTestCase {
             //end::migration-request-ctor-execution
             assertEquals(RestStatus.CREATED, response.status());
         }
-        {
-            //tag::migration-request-sync-execution
-            DeleteRequest request = new DeleteRequest("index", "doc", "id");
-            DeleteResponse response = client.delete(request); // <1>
-            //end::migration-request-sync-execution
-            assertEquals(RestStatus.OK, response.status());
-        }
         {
             //tag::migration-request-async-execution
             DeleteRequest request = new DeleteRequest("index", "doc", "id"); // <1>
@@ -155,6 +151,14 @@ public class MigrationDocumentationIT extends ESRestHighLevelClientTestCase {
                 }
             });
             //end::migration-request-async-execution
+            assertBusy(() -> assertFalse(client.exists(new GetRequest("index", "doc", "id"))));
+        }
+        {
+            //tag::migration-request-sync-execution
+            DeleteRequest request = new DeleteRequest("index", "doc", "id");
+            DeleteResponse response = client.delete(request); // <1>
+            //end::migration-request-sync-execution
+            assertEquals(RestStatus.NOT_FOUND, response.status());
         }
     }
 }
diff --git a/docs/java-rest/high-level/migration.asciidoc b/docs/java-rest/high-level/migration.asciidoc
index 7ce0b00bdb0..44e895c9c71 100644
--- a/docs/java-rest/high-level/migration.asciidoc
+++ b/docs/java-rest/high-level/migration.asciidoc
@@ -291,7 +291,7 @@ The same operation executed with the low-level client could be:
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
-include-tagged::{doc-tests}/MigrationDocumentationIT.java[migration-create-inded]
+include-tagged::{doc-tests}/MigrationDocumentationIT.java[migration-create-index]
 --------------------------------------------------
 <1> Define the settings of the index
 <2> Define the body of the HTTP request using a `XContentBuilder` with JSON format
@@ -331,8 +331,8 @@ With the low-level client, the code can be changed to:
 --------------------------------------------------
 include-tagged::{doc-tests}/MigrationDocumentationIT.java[migration-cluster-health]
 --------------------------------------------------
-<1> Call the cluster's health REST endpoint using the default paramaters
-and gets back a `Response` object.
+<1> Call the cluster's health REST endpoint and wait for the cluster health to become green,
+then get back a `Response` object.
 <2> Retrieve an `InputStream` object in order to read the response's content
 <3> Parse the response's content using Elasticsearch's helper class `XContentHelper`. This
  helper requires the content type of the response to be passed as an argument and returns

From 442c3b8bcf3a8157d8b7fe6d5f6319ef8416e3ab Mon Sep 17 00:00:00 2001
From: Martijn van Groningen 
Date: Wed, 13 Dec 2017 16:50:49 +0100
Subject: [PATCH 257/297] docs: fix link

---
 .../analysis/tokenfilters/normalization-tokenfilter.asciidoc    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/analysis/tokenfilters/normalization-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/normalization-tokenfilter.asciidoc
index a2f0474b02e..2ff8ab13497 100644
--- a/docs/reference/analysis/tokenfilters/normalization-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/normalization-tokenfilter.asciidoc
@@ -36,5 +36,5 @@ http://lucene.apache.org/core/4_9_0/analyzers-common/org/apache/lucene/analysis/
 
 Serbian::
 
-not-released-yet[`serbian_normalization`],
+http://lucene.apache.org/core/7_1_0/analyzers-common/org/apache/lucene/analysis/sr/SerbianNormalizationFilter.html[`serbian_normalization`]
 

From 5bc2f390a5d0e3eeef29a4f242a474bfc34af696 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen 
Date: Wed, 13 Dec 2017 11:10:57 -0500
Subject: [PATCH 258/297] Use CountedBitSet in LocalCheckpointTracker (#27793)

The CountedBitSet can automatically release its internal bitsets when
all bits are set to reduce memory usage. This structure can work well
for sequence numbers as these numbers are likely to form contiguous
ranges. This commit replaces FixedBitSet by CountedBitSet in
LocalCheckpointTracker.
---
 .../{translog => seqno}/CountedBitSet.java     | 10 +++++-----
 .../index/seqno/LocalCheckpointTracker.java    | 18 +++++++++---------
 .../index/translog/MultiSnapshot.java          |  1 +
 .../CountedBitSetTests.java                    |  2 +-
 .../seqno/LocalCheckpointTrackerTests.java     |  4 ++--
 5 files changed, 18 insertions(+), 17 deletions(-)
 rename core/src/main/java/org/elasticsearch/index/{translog => seqno}/CountedBitSet.java (93%)
 rename core/src/test/java/org/elasticsearch/index/{translog => seqno}/CountedBitSetTests.java (99%)

diff --git a/core/src/main/java/org/elasticsearch/index/translog/CountedBitSet.java b/core/src/main/java/org/elasticsearch/index/seqno/CountedBitSet.java
similarity index 93%
rename from core/src/main/java/org/elasticsearch/index/translog/CountedBitSet.java
rename to core/src/main/java/org/elasticsearch/index/seqno/CountedBitSet.java
index ca1ae279a99..54270de1b01 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/CountedBitSet.java
+++ b/core/src/main/java/org/elasticsearch/index/seqno/CountedBitSet.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.translog;
+package org.elasticsearch.index.seqno;
 
 import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.FixedBitSet;
@@ -25,15 +25,15 @@ import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * A {@link CountedBitSet} wraps a {@link FixedBitSet} but automatically releases the internal bitset
- * when all bits are set to reduce memory usage. This structure can work well for sequence numbers
- * from translog as these numbers are likely to form contiguous ranges (eg. filling all bits).
+ * when all bits are set to reduce memory usage. This structure can work well for sequence numbers as
+ * these numbers are likely to form contiguous ranges (eg. filling all bits).
  */
-final class CountedBitSet extends BitSet {
+public final class CountedBitSet extends BitSet {
     static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(CountedBitSet.class);
     private short onBits; // Number of bits are set.
     private FixedBitSet bitset;
 
-    CountedBitSet(short numBits) {
+    public CountedBitSet(short numBits) {
         if (numBits <= 0) {
             throw new IllegalArgumentException("Number of bits must be positive. Given [" + numBits + "]");
         }
diff --git a/core/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java b/core/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java
index 54751e8958a..2644dbfc32d 100644
--- a/core/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java
+++ b/core/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.index.seqno;
 
 import com.carrotsearch.hppc.LongObjectHashMap;
-import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.BitSet;
 import org.elasticsearch.common.SuppressForbidden;
 
 /**
@@ -33,13 +33,13 @@ public class LocalCheckpointTracker {
      * We keep a bit for each sequence number that is still pending. To optimize allocation, we do so in multiple sets allocating them on
      * demand and cleaning up while completed. This constant controls the size of the sets.
      */
-    static final int BIT_SET_SIZE = 1024;
+    static final short BIT_SET_SIZE = 1024;
 
     /**
      * A collection of bit sets representing pending sequence numbers. Each sequence number is mapped to a bit set by dividing by the
      * bit set size.
      */
-    final LongObjectHashMap processedSeqNo = new LongObjectHashMap<>();
+    final LongObjectHashMap processedSeqNo = new LongObjectHashMap<>();
 
     /**
      * The current local checkpoint, i.e., all sequence numbers no more than this number have been completed.
@@ -96,7 +96,7 @@ public class LocalCheckpointTracker {
             // this is possible during recovery where we might replay an operation that was also replicated
             return;
         }
-        final FixedBitSet bitSet = getBitSetForSeqNo(seqNo);
+        final BitSet bitSet = getBitSetForSeqNo(seqNo);
         final int offset = seqNoToBitSetOffset(seqNo);
         bitSet.set(offset);
         if (seqNo == checkpoint + 1) {
@@ -170,7 +170,7 @@ public class LocalCheckpointTracker {
         try {
             // keep it simple for now, get the checkpoint one by one; in the future we can optimize and read words
             long bitSetKey = getBitSetKey(checkpoint);
-            FixedBitSet current = processedSeqNo.get(bitSetKey);
+            BitSet current = processedSeqNo.get(bitSetKey);
             if (current == null) {
                 // the bit set corresponding to the checkpoint has already been removed, set ourselves up for the next bit set
                 assert checkpoint % BIT_SET_SIZE == BIT_SET_SIZE - 1;
@@ -184,7 +184,7 @@ public class LocalCheckpointTracker {
                  */
                 if (checkpoint == lastSeqNoInBitSet(bitSetKey)) {
                     assert current != null;
-                    final FixedBitSet removed = processedSeqNo.remove(bitSetKey);
+                    final BitSet removed = processedSeqNo.remove(bitSetKey);
                     assert removed == current;
                     current = processedSeqNo.get(++bitSetKey);
                 }
@@ -210,15 +210,15 @@ public class LocalCheckpointTracker {
         return seqNo / BIT_SET_SIZE;
     }
 
-    private FixedBitSet getBitSetForSeqNo(final long seqNo) {
+    private BitSet getBitSetForSeqNo(final long seqNo) {
         assert Thread.holdsLock(this);
         final long bitSetKey = getBitSetKey(seqNo);
         final int index = processedSeqNo.indexOf(bitSetKey);
-        final FixedBitSet bitSet;
+        final BitSet bitSet;
         if (processedSeqNo.indexExists(index)) {
             bitSet = processedSeqNo.indexGet(index);
         } else {
-            bitSet = new FixedBitSet(BIT_SET_SIZE);
+            bitSet = new CountedBitSet(BIT_SET_SIZE);
             processedSeqNo.indexInsert(index, bitSetKey, bitSet);
         }
         return bitSet;
diff --git a/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java b/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java
index e4bfbdcd42e..910d71a51a0 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.translog;
 
 import com.carrotsearch.hppc.LongObjectHashMap;
 import org.apache.lucene.util.BitSet;
+import org.elasticsearch.index.seqno.CountedBitSet;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 
 import java.io.Closeable;
diff --git a/core/src/test/java/org/elasticsearch/index/translog/CountedBitSetTests.java b/core/src/test/java/org/elasticsearch/index/seqno/CountedBitSetTests.java
similarity index 99%
rename from core/src/test/java/org/elasticsearch/index/translog/CountedBitSetTests.java
rename to core/src/test/java/org/elasticsearch/index/seqno/CountedBitSetTests.java
index b68607f02d6..b014f827406 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/CountedBitSetTests.java
+++ b/core/src/test/java/org/elasticsearch/index/seqno/CountedBitSetTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.translog;
+package org.elasticsearch.index.seqno;
 
 import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.test.ESTestCase;
diff --git a/core/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java b/core/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java
index eb62391e0b0..31b8c23bf1c 100644
--- a/core/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java
+++ b/core/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.index.seqno;
 
 import com.carrotsearch.hppc.LongObjectHashMap;
-import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.BitSet;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
@@ -260,7 +260,7 @@ public class LocalCheckpointTrackerTests extends ESTestCase {
         tracker.resetCheckpoint(localCheckpoint);
         assertThat(tracker.getCheckpoint(), equalTo((long) localCheckpoint));
         assertThat(tracker.getMaxSeqNo(), equalTo((long) maxSeqNo));
-        assertThat(tracker.processedSeqNo, new BaseMatcher>() {
+        assertThat(tracker.processedSeqNo, new BaseMatcher>() {
             @Override
             public boolean matches(Object item) {
                 return (item instanceof LongObjectHashMap && ((LongObjectHashMap) item).isEmpty());

From 94cfc2a0df7747b62daa527d65ea02cbcd65aa7f Mon Sep 17 00:00:00 2001
From: Glen Smith 
Date: Wed, 13 Dec 2017 09:20:12 -0700
Subject: [PATCH 259/297] [Docs] Fix explanation of
 "cluster.routing.allocation.exclude" (#27735)

---
 docs/reference/modules/cluster/allocation_filtering.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/modules/cluster/allocation_filtering.asciidoc b/docs/reference/modules/cluster/allocation_filtering.asciidoc
index c173e4eb385..94c653ec4e1 100644
--- a/docs/reference/modules/cluster/allocation_filtering.asciidoc
+++ b/docs/reference/modules/cluster/allocation_filtering.asciidoc
@@ -21,7 +21,7 @@ refers to an arbitrary node attribute.:
 
 `cluster.routing.allocation.exclude.{attribute}`::
 
-    Do not allocate shards to a node whose `{attribute}` has _none_ of the
+    Do not allocate shards to a node whose `{attribute}` has _any_ of the
     comma-separated values.
 
 These special attributes are also supported:

From 5406a9f30dc5f391b47463f4e5da2064a6298e66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= 
Date: Wed, 13 Dec 2017 16:26:47 +0100
Subject: [PATCH 260/297] Add rank-eval module to transport client and HL
 client dependencies

---
 build.gradle                                  |  1 +
 client/rest-high-level/build.gradle           |  3 +-
 .../client/RestHighLevelClientTests.java      | 40 +++++++++++--------
 client/transport/build.gradle                 |  3 +-
 docs/Versions.asciidoc                        |  2 +
 5 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/build.gradle b/build.gradle
index 99404c15e3a..9feeed809ae 100644
--- a/build.gradle
+++ b/build.gradle
@@ -196,6 +196,7 @@ subprojects {
     "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
     "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
+    "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
   ]
 
   for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
index ba97605dba8..c273e76a92a 100644
--- a/client/rest-high-level/build.gradle
+++ b/client/rest-high-level/build.gradle
@@ -39,6 +39,7 @@ dependencies {
   compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
   compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
+  compile "org.elasticsearch.plugin:rank-eval-client:${version}"
 
   testCompile "org.elasticsearch.client:test:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
@@ -60,4 +61,4 @@ forbiddenApisMain {
   // specified
   signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
   signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
-}
\ No newline at end of file
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index 1ab4c60d624..1d5961c506f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -21,20 +21,6 @@ package org.elasticsearch.client;
 
 import com.fasterxml.jackson.core.JsonParseException;
 
-import org.elasticsearch.Build;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.main.MainRequest;
-import org.elasticsearch.action.main.MainResponse;
-import org.elasticsearch.action.search.ClearScrollRequest;
-import org.elasticsearch.action.search.ClearScrollResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchResponseSections;
-import org.elasticsearch.action.search.SearchScrollRequest;
-import org.elasticsearch.action.search.ShardSearchFailure;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
@@ -49,6 +35,20 @@ import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.message.BasicRequestLine;
 import org.apache.http.message.BasicStatusLine;
 import org.apache.http.nio.entity.NStringEntity;
+import org.elasticsearch.Build;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.main.MainRequest;
+import org.elasticsearch.action.main.MainResponse;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchResponseSections;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -57,6 +57,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.cbor.CborXContent;
 import org.elasticsearch.common.xcontent.smile.SmileXContent;
+import org.elasticsearch.index.rankeval.DiscountedCumulativeGain;
+import org.elasticsearch.index.rankeval.EvaluationMetric;
+import org.elasticsearch.index.rankeval.MeanReciprocalRank;
+import org.elasticsearch.index.rankeval.PrecisionAtK;
 import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHits;
@@ -648,7 +652,7 @@ public class RestHighLevelClientTests extends ESTestCase {
 
     public void testProvidedNamedXContents() {
         List namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(2, namedXContents.size());
+        assertEquals(5, namedXContents.size());
         Map, Integer> categories = new HashMap<>();
         List names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -658,10 +662,14 @@ public class RestHighLevelClientTests extends ESTestCase {
                 categories.put(namedXContent.categoryClass, counter + 1);
             }
         }
-        assertEquals(1, categories.size());
+        assertEquals(2, categories.size());
         assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
         assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
         assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
+        assertEquals(Integer.valueOf(3), categories.get(EvaluationMetric.class));
+        assertTrue(names.contains(PrecisionAtK.NAME));
+        assertTrue(names.contains(DiscountedCumulativeGain.NAME));
+        assertTrue(names.contains(MeanReciprocalRank.NAME));
     }
 
     private static class TrackingActionListener implements ActionListener {
diff --git a/client/transport/build.gradle b/client/transport/build.gradle
index f09668ac6ac..944a038edd9 100644
--- a/client/transport/build.gradle
+++ b/client/transport/build.gradle
@@ -32,6 +32,7 @@ dependencies {
   compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
   compile "org.elasticsearch.plugin:percolator-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
+  compile "org.elasticsearch.plugin:rank-eval-client:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
@@ -54,4 +55,4 @@ namingConventions {
   testClass = 'com.carrotsearch.randomizedtesting.RandomizedTest'
   //we don't have integration tests
   skipIntegTestInDisguise = true
-}
\ No newline at end of file
+}
diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index 7f6d952deca..1188e6268e3 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -36,6 +36,7 @@ ifeval::["{release-state}"=="unreleased"]
 :parent-join-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}-SNAPSHOT
 :percolator-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}-SNAPSHOT
 :matrixstats-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/aggs-matrix-stats-client/{version}-SNAPSHOT
+:rank-eval-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/rank-eval-client/{version}-SNAPSHOT
 endif::[]
 
 ifeval::["{release-state}"!="unreleased"]
@@ -48,6 +49,7 @@ ifeval::["{release-state}"!="unreleased"]
 :parent-join-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}
 :percolator-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}
 :matrixstats-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/aggs-matrix-stats-client/{version}
+:rank-eval-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/rank-eval-client/{version}
 endif::[]
 
 ///////

From 0c5086af584131b8d8228c1470bb94b59ed6b203 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer 
Date: Thu, 14 Dec 2017 09:22:09 +0100
Subject: [PATCH 261/297] Add unreleased v6.1.1 version

---
 core/src/main/java/org/elasticsearch/Version.java | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index cfdff404d32..ea74a284d87 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -138,6 +138,8 @@ public class Version implements Comparable {
         new Version(V_6_0_2_ID, org.apache.lucene.util.Version.LUCENE_7_0_1);
     public static final int V_6_1_0_ID = 6010099;
     public static final Version V_6_1_0 = new Version(V_6_1_0_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
+    public static final int V_6_1_1_ID = 6010199;
+    public static final Version V_6_1_1 = new Version(V_6_1_1_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
     public static final int V_6_2_0_ID = 6020099;
     public static final Version V_6_2_0 = new Version(V_6_2_0_ID, org.apache.lucene.util.Version.LUCENE_7_2_0);
     public static final int V_7_0_0_alpha1_ID = 7000001;
@@ -158,10 +160,12 @@ public class Version implements Comparable {
         switch (id) {
             case V_7_0_0_alpha1_ID:
                 return V_7_0_0_alpha1;
-            case V_6_1_0_ID:
-                return V_6_1_0;
             case V_6_2_0_ID:
                 return V_6_2_0;
+            case V_6_1_1_ID:
+                return V_6_1_1;
+            case V_6_1_0_ID:
+                return V_6_1_0;
             case V_6_0_2_ID:
                 return V_6_0_2;
             case V_6_0_1_ID:

From 7e0fc8a112b5797c4f4e8051236b95c541c935c2 Mon Sep 17 00:00:00 2001
From: Sandeep Kanabar 
Date: Thu, 14 Dec 2017 14:42:31 +0530
Subject: [PATCH 262/297] [Docs] Correct spelling in update-settings.asciidoc
 (#27808)

---
 docs/reference/cluster/update-settings.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/cluster/update-settings.asciidoc b/docs/reference/cluster/update-settings.asciidoc
index 7cf47a99f2f..57be634e835 100644
--- a/docs/reference/cluster/update-settings.asciidoc
+++ b/docs/reference/cluster/update-settings.asciidoc
@@ -2,7 +2,7 @@
 == Cluster Update Settings
 
 Allows to update cluster wide specific settings. Settings updated can
-either be persistent (applied cross restarts) or transient (will not
+either be persistent (applied across restarts) or transient (will not
 survive a full cluster restart). Here is an example:
 
 [source,js]

From d26b33dea2be5386a0f8479bc1d71d83692206cb Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer 
Date: Thu, 14 Dec 2017 12:33:41 +0100
Subject: [PATCH 263/297] Mute
 VersionUtilsTest#testGradleVersionsMatchVersionUtils

Relates #27815
---
 .../test/java/org/elasticsearch/test/VersionUtilsTests.java   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
index 3be5ec5913d..7c453c895c6 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
@@ -259,6 +259,10 @@ public class VersionUtilsTests extends ESTestCase {
      * agree with the list of wire and index compatible versions we build in gradle.
      */
     public void testGradleVersionsMatchVersionUtils() {
+        if (System.getProperty("build.snapshot", "true").equals("false")) {
+            logger.warn("Skipping testGradleVersionsMatchVersionUtils(): See #27815 for details");
+            return;
+        }
         // First check the index compatible versions
         VersionsFromProperty indexCompatible = new VersionsFromProperty("tests.gradle_index_compat_versions");
         List released = VersionUtils.allReleasedVersions().stream()

From 579d1fea5738043e28e78a9d50ca0052fbd5eb30 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe 
Date: Thu, 14 Dec 2017 12:17:17 +0000
Subject: [PATCH 264/297] Fixes ByteSizeValue to serialise correctly (#27702)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Fixes ByteSizeValue to serialise correctly

This fix makes a few fixes to ByteSizeValue to make it possible to perform round-trip serialisation:
* Changes wire serialisation to use Zlong methods instead of VLong methods. This is needed because the value `-1` is accepted but previously if `-1` is supplied it cannot be serialised using the wire protocol.
* Limits the supplied size to be no more than Long.MAX_VALUE when converted to bytes. Previously values greater than Long.MAX_VALUE bytes were accepted but would be silently interpreted as Long.MAX_VALUE bytes rather than erroring so the user had no idea the value was not being used the way they had intended. I consider this a bug and so fine to include this bug fix in a minor version but I am open to other points of view.
* Adds a `getStringRep()` method that can be used when serialising the value to JSON. This will print the bytes value if the size is positive, `”0”` if the size is `0` and `”-1”` if the size is `-1`.
* Adds logic to detect fractional values when parsing from a String and emits a deprecation warning in this case.
* Modifies hashCode and equals methods to work with long values rather than doubles so they don’t run into precision problems when dealing with large values. Previous to this change the equals method would not detect small differences in the values (e.g. 1-1000 bytes ranges) if the actual values where very large (e.g. PBs). This was due to the values being in the order of 10^18 but doubles only maintaining a precision of ~10^15.

Closes #27568

* Fix bytes settings default value to not use fractional values

* Fixes test

* Addresses review comments

* Modifies parsing to preserve unit

This should be bwc since in the case that the input is fractional it reverts back to the old method of parsing it to the bytes value.

* Addresses more review comments

* Fixes tests

* Temporarily changes version check to 7.0.0

This will be changed to 6.2 when the fix has been backported
---
 .../common/settings/Setting.java              |   3 +-
 .../common/unit/ByteSizeUnit.java             |  32 ++++
 .../common/unit/ByteSizeValue.java            | 170 ++++++++++++------
 .../cluster/node/stats/NodeStatsTests.java    |   6 +-
 .../elasticsearch/cluster/DiskUsageTests.java |   7 +-
 .../common/unit/ByteSizeValueTests.java       | 153 ++++++++++++++--
 .../index/IndexSettingsTests.java             |  18 +-
 .../IndexingMemoryControllerTests.java        |   6 +-
 .../monitor/os/OsStatsTests.java              |   6 +-
 .../repositories/s3/S3RepositoryTests.java    |  17 +-
 10 files changed, 316 insertions(+), 102 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
index 522bbdc9f35..bc22dbb63eb 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -52,7 +52,6 @@ import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import java.util.function.IntConsumer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -1020,7 +1019,7 @@ public class Setting implements ToXContentObject {
 
     public static Setting byteSizeSetting(String key, ByteSizeValue defaultValue, ByteSizeValue minValue,
                                                          ByteSizeValue maxValue, Property... properties) {
-        return byteSizeSetting(key, (s) -> defaultValue.toString(), minValue, maxValue, properties);
+        return byteSizeSetting(key, (s) -> defaultValue.getStringRep(), minValue, maxValue, properties);
     }
 
     public static Setting byteSizeSetting(String key, Function defaultValue,
diff --git a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeUnit.java b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeUnit.java
index e7e43b6d78a..ec6de176c2f 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeUnit.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeUnit.java
@@ -63,6 +63,11 @@ public enum ByteSizeUnit implements Writeable {
         public long toPB(long size) {
             return size / (C5 / C0);
         }
+
+        @Override
+        public String getSuffix() {
+            return "b";
+        }
     },
     KB {
         @Override
@@ -94,6 +99,11 @@ public enum ByteSizeUnit implements Writeable {
         public long toPB(long size) {
             return size / (C5 / C1);
         }
+
+        @Override
+        public String getSuffix() {
+            return "kb";
+        }
     },
     MB {
         @Override
@@ -125,6 +135,11 @@ public enum ByteSizeUnit implements Writeable {
         public long toPB(long size) {
             return size / (C5 / C2);
         }
+
+        @Override
+        public String getSuffix() {
+            return "mb";
+        }
     },
     GB {
         @Override
@@ -156,6 +171,11 @@ public enum ByteSizeUnit implements Writeable {
         public long toPB(long size) {
             return size / (C5 / C3);
         }
+
+        @Override
+        public String getSuffix() {
+            return "gb";
+        }
     },
     TB {
         @Override
@@ -187,6 +207,11 @@ public enum ByteSizeUnit implements Writeable {
         public long toPB(long size) {
             return size / (C5 / C4);
         }
+
+        @Override
+        public String getSuffix() {
+            return "tb";
+        }
     },
     PB {
         @Override
@@ -218,6 +243,11 @@ public enum ByteSizeUnit implements Writeable {
         public long toPB(long size) {
             return size;
         }
+
+        @Override
+        public String getSuffix() {
+            return "pb";
+        }
     };
 
     static final long C0 = 1L;
@@ -258,6 +288,8 @@ public enum ByteSizeUnit implements Writeable {
 
     public abstract long toPB(long size);
 
+    public abstract String getSuffix();
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(this.ordinal());
diff --git a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
index e0782e32cae..9fb7a3852d7 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
@@ -20,28 +20,42 @@
 package org.elasticsearch.common.unit;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 
 import java.io.IOException;
 import java.util.Locale;
 import java.util.Objects;
 
 public class ByteSizeValue implements Writeable, Comparable {
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(ByteSizeValue.class));
 
     private final long size;
     private final ByteSizeUnit unit;
 
     public ByteSizeValue(StreamInput in) throws IOException {
-        size = in.readVLong();
-        unit = ByteSizeUnit.BYTES;
+        if (in.getVersion().before(Version.V_7_0_0_alpha1)) {
+            size = in.readVLong();
+            unit = ByteSizeUnit.BYTES;
+        } else {
+            size = in.readZLong();
+            unit = ByteSizeUnit.readFrom(in);
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVLong(getBytes());
+        if (out.getVersion().before(Version.V_7_0_0_alpha1)) {
+            out.writeVLong(getBytes());
+        } else {
+            out.writeZLong(size);
+            unit.writeTo(out);
+        }
     }
 
     public ByteSizeValue(long bytes) {
@@ -49,10 +63,28 @@ public class ByteSizeValue implements Writeable, Comparable {
     }
 
     public ByteSizeValue(long size, ByteSizeUnit unit) {
+        if (size < -1 || (size == -1 && unit != ByteSizeUnit.BYTES)) {
+            throw new IllegalArgumentException("Values less than -1 bytes are not supported: " + size + unit.getSuffix());
+        }
+        if (size > Long.MAX_VALUE / unit.toBytes(1)) {
+            throw new IllegalArgumentException(
+                    "Values greater than " + Long.MAX_VALUE + " bytes are not supported: " + size + unit.getSuffix());
+        }
         this.size = size;
         this.unit = unit;
     }
 
+    // For testing
+    long getSize() {
+        return size;
+    }
+
+    // For testing
+    ByteSizeUnit getUnit() {
+        return unit;
+    }
+
+    @Deprecated
     public int bytesAsInt() {
         long bytes = getBytes();
         if (bytes > Integer.MAX_VALUE) {
@@ -105,26 +137,41 @@ public class ByteSizeValue implements Writeable, Comparable {
         return ((double) getBytes()) / ByteSizeUnit.C5;
     }
 
+    /**
+     * @return a string representation of this value which is guaranteed to be
+     *         able to be parsed using
+     *         {@link #parseBytesSizeValue(String, ByteSizeValue, String)}.
+     *         Unlike {@link #toString()} this method will not output fractional
+     *         or rounded values so this method should be preferred when
+     *         serialising the value to JSON.
+     */
+    public String getStringRep() {
+        if (size <= 0) {
+            return String.valueOf(size);
+        }
+        return size + unit.getSuffix();
+    }
+
     @Override
     public String toString() {
         long bytes = getBytes();
         double value = bytes;
-        String suffix = "b";
+        String suffix = ByteSizeUnit.BYTES.getSuffix();
         if (bytes >= ByteSizeUnit.C5) {
             value = getPbFrac();
-            suffix = "pb";
+            suffix = ByteSizeUnit.PB.getSuffix();
         } else if (bytes >= ByteSizeUnit.C4) {
             value = getTbFrac();
-            suffix = "tb";
+            suffix = ByteSizeUnit.TB.getSuffix();
         } else if (bytes >= ByteSizeUnit.C3) {
             value = getGbFrac();
-            suffix = "gb";
+            suffix = ByteSizeUnit.GB.getSuffix();
         } else if (bytes >= ByteSizeUnit.C2) {
             value = getMbFrac();
-            suffix = "mb";
+            suffix = ByteSizeUnit.MB.getSuffix();
         } else if (bytes >= ByteSizeUnit.C1) {
             value = getKbFrac();
-            suffix = "kb";
+            suffix = ByteSizeUnit.KB.getSuffix();
         }
         return Strings.format1Decimals(value, suffix);
     }
@@ -139,47 +186,64 @@ public class ByteSizeValue implements Writeable, Comparable {
         if (sValue == null) {
             return defaultValue;
         }
-        long bytes;
-        try {
-            String lowerSValue = sValue.toLowerCase(Locale.ROOT).trim();
-            if (lowerSValue.endsWith("k")) {
-                bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * ByteSizeUnit.C1);
-            } else if (lowerSValue.endsWith("kb")) {
-                bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) * ByteSizeUnit.C1);
-            } else if (lowerSValue.endsWith("m")) {
-                bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * ByteSizeUnit.C2);
-            } else if (lowerSValue.endsWith("mb")) {
-                bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) * ByteSizeUnit.C2);
-            } else if (lowerSValue.endsWith("g")) {
-                bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * ByteSizeUnit.C3);
-            } else if (lowerSValue.endsWith("gb")) {
-                bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) * ByteSizeUnit.C3);
-            } else if (lowerSValue.endsWith("t")) {
-                bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * ByteSizeUnit.C4);
-            } else if (lowerSValue.endsWith("tb")) {
-                bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) * ByteSizeUnit.C4);
-            } else if (lowerSValue.endsWith("p")) {
-                bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * ByteSizeUnit.C5);
-            } else if (lowerSValue.endsWith("pb")) {
-                bytes = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2)) * ByteSizeUnit.C5);
-            } else if (lowerSValue.endsWith("b")) {
-                bytes = Long.parseLong(lowerSValue.substring(0, lowerSValue.length() - 1).trim());
-            } else if (lowerSValue.equals("-1")) {
-                // Allow this special value to be unit-less:
-                bytes = -1;
-            } else if (lowerSValue.equals("0")) {
-                // Allow this special value to be unit-less:
-                bytes = 0;
-            } else {
-                // Missing units:
-                throw new ElasticsearchParseException(
-                        "failed to parse setting [{}] with value [{}] as a size in bytes: unit is missing or unrecognized",
-                        settingName, sValue);
-            }
-        } catch (NumberFormatException e) {
-            throw new ElasticsearchParseException("failed to parse [{}]", e, sValue);
+        String lowerSValue = sValue.toLowerCase(Locale.ROOT).trim();
+        if (lowerSValue.endsWith("k")) {
+            return parse(sValue, lowerSValue, "k", ByteSizeUnit.KB, settingName);
+        } else if (lowerSValue.endsWith("kb")) {
+            return parse(sValue, lowerSValue, "kb", ByteSizeUnit.KB, settingName);
+        } else if (lowerSValue.endsWith("m")) {
+            return parse(sValue, lowerSValue, "m", ByteSizeUnit.MB, settingName);
+        } else if (lowerSValue.endsWith("mb")) {
+            return parse(sValue, lowerSValue, "mb", ByteSizeUnit.MB, settingName);
+        } else if (lowerSValue.endsWith("g")) {
+            return parse(sValue, lowerSValue, "g", ByteSizeUnit.GB, settingName);
+        } else if (lowerSValue.endsWith("gb")) {
+            return parse(sValue, lowerSValue, "gb", ByteSizeUnit.GB, settingName);
+        } else if (lowerSValue.endsWith("t")) {
+            return parse(sValue, lowerSValue, "t", ByteSizeUnit.TB, settingName);
+        } else if (lowerSValue.endsWith("tb")) {
+            return parse(sValue, lowerSValue, "tb", ByteSizeUnit.TB, settingName);
+        } else if (lowerSValue.endsWith("p")) {
+            return parse(sValue, lowerSValue, "p", ByteSizeUnit.PB, settingName);
+        } else if (lowerSValue.endsWith("pb")) {
+            return parse(sValue, lowerSValue, "pb", ByteSizeUnit.PB, settingName);
+        } else if (lowerSValue.endsWith("b")) {
+            return new ByteSizeValue(Long.parseLong(lowerSValue.substring(0, lowerSValue.length() - 1).trim()), ByteSizeUnit.BYTES);
+        } else if (lowerSValue.equals("-1")) {
+            // Allow this special value to be unit-less:
+            return new ByteSizeValue(-1, ByteSizeUnit.BYTES);
+        } else if (lowerSValue.equals("0")) {
+            // Allow this special value to be unit-less:
+            return new ByteSizeValue(0, ByteSizeUnit.BYTES);
+        } else {
+            // Missing units:
+            throw new ElasticsearchParseException(
+                    "failed to parse setting [{}] with value [{}] as a size in bytes: unit is missing or unrecognized", settingName,
+                    sValue);
+        }
+    }
+
+    private static ByteSizeValue parse(final String initialInput, final String normalized, final String suffix, ByteSizeUnit unit,
+            final String settingName) {
+        final String s = normalized.substring(0, normalized.length() - suffix.length()).trim();
+        try {
+            try {
+                return new ByteSizeValue(Long.parseLong(s), unit);
+            } catch (final NumberFormatException e) {
+                try {
+                    final double doubleValue = Double.parseDouble(s);
+                    DEPRECATION_LOGGER.deprecated(
+                            "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [{}] found for setting [{}]",
+                            initialInput, settingName);
+                    return new ByteSizeValue((long) (doubleValue * unit.toBytes(1)));
+                } catch (final NumberFormatException ignored) {
+                    throw new ElasticsearchParseException("failed to parse [{}]", e, initialInput);
+                }
+            }
+        } catch (IllegalArgumentException e) {
+            throw new ElasticsearchParseException("failed to parse setting [{}] with value [{}] as a size in bytes", e, settingName,
+                    initialInput);
         }
-        return new ByteSizeValue(bytes, ByteSizeUnit.BYTES);
     }
 
     @Override
@@ -196,13 +260,13 @@ public class ByteSizeValue implements Writeable, Comparable {
 
     @Override
     public int hashCode() {
-        return Double.hashCode(((double) size) * unit.toBytes(1));
+        return Long.hashCode(size * unit.toBytes(1));
     }
 
     @Override
     public int compareTo(ByteSizeValue other) {
-        double thisValue = ((double) size) * unit.toBytes(1);
-        double otherValue = ((double) other.size) * other.unit.toBytes(1);
-        return Double.compare(thisValue, otherValue);
+        long thisValue = size * unit.toBytes(1);
+        long otherValue = other.size * other.unit.toBytes(1);
+        return Long.compare(thisValue, otherValue);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
index d9aed454732..7bf43b828c0 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -311,9 +311,11 @@ public class NodeStatsTests extends ESTestCase {
             for (int i = 0; i < 3; i++) {
                 loadAverages[i] = randomBoolean() ? randomDouble() : -1;
             }
+            long memTotal = randomNonNegativeLong();
+            long swapTotal = randomNonNegativeLong();
             osStats = new OsStats(System.currentTimeMillis(), new OsStats.Cpu(randomShort(), loadAverages),
-                    new OsStats.Mem(randomLong(), randomLong()),
-                    new OsStats.Swap(randomLong(), randomLong()),
+                    new OsStats.Mem(memTotal, randomLongBetween(0, memTotal)),
+                    new OsStats.Swap(swapTotal, randomLongBetween(0, swapTotal)),
                     new OsStats.Cgroup(
                         randomAlphaOfLength(8),
                         randomNonNegativeLong(),
diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
index 09ff06919e9..4ef8f7cbdb7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
@@ -23,15 +23,12 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardPath;
@@ -184,12 +181,12 @@ public class DiskUsageTests extends ESTestCase {
                 new FsInfo.Path("/most", "/dev/sdc", 300, 290, 280),
         };
         FsInfo.Path[] node2FSInfo = new FsInfo.Path[] {
-                new FsInfo.Path("/least_most", "/dev/sda", -2, -1, -1),
+                new FsInfo.Path("/least_most", "/dev/sda", -1, -1, -1),
         };
 
         FsInfo.Path[] node3FSInfo =  new FsInfo.Path[] {
                 new FsInfo.Path("/most", "/dev/sda", 100, 90, 70),
-                new FsInfo.Path("/least", "/dev/sda", 10, -8, 0),
+                new FsInfo.Path("/least", "/dev/sda", 10, -1, 0),
         };
         List nodeStats = Arrays.asList(
                 new NodeStats(new DiscoveryNode("node_1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0,
diff --git a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
index 9d0a3e67e9d..d9010136ca0 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.common.unit;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
@@ -31,7 +31,7 @@ import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
-public class ByteSizeValueTests extends ESTestCase {
+public class ByteSizeValueTests extends AbstractWireSerializingTestCase {
     public void testActualPeta() {
         MatcherAssert.assertThat(new ByteSizeValue(4, ByteSizeUnit.PB).getBytes(), equalTo(4503599627370496L));
     }
@@ -150,17 +150,17 @@ public class ByteSizeValueTests extends ESTestCase {
     }
 
     public void testCompareEquality() {
-        long firstRandom = randomNonNegativeLong();
         ByteSizeUnit randomUnit = randomFrom(ByteSizeUnit.values());
+        long firstRandom = randomNonNegativeLong() / randomUnit.toBytes(1);
         ByteSizeValue firstByteValue = new ByteSizeValue(firstRandom, randomUnit);
         ByteSizeValue secondByteValue = new ByteSizeValue(firstRandom, randomUnit);
         assertEquals(0, firstByteValue.compareTo(secondByteValue));
     }
 
     public void testCompareValue() {
-        long firstRandom = randomNonNegativeLong();
-        long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomNonNegativeLong);
         ByteSizeUnit unit = randomFrom(ByteSizeUnit.values());
+        long firstRandom = randomNonNegativeLong() / unit.toBytes(1);
+        long secondRandom = randomValueOtherThan(firstRandom, () -> randomNonNegativeLong() / unit.toBytes(1));
         ByteSizeValue firstByteValue = new ByteSizeValue(firstRandom, unit);
         ByteSizeValue secondByteValue = new ByteSizeValue(secondRandom, unit);
         assertEquals(firstRandom > secondRandom, firstByteValue.compareTo(secondByteValue) > 0);
@@ -168,7 +168,7 @@ public class ByteSizeValueTests extends ESTestCase {
     }
 
     public void testCompareUnits() {
-        long number = randomNonNegativeLong();
+        long number = randomNonNegativeLong() / ByteSizeUnit.PB.toBytes(1);
         ByteSizeUnit randomUnit = randomValueOtherThan(ByteSizeUnit.PB, ()->randomFrom(ByteSizeUnit.values()));
         ByteSizeValue firstByteValue = new ByteSizeValue(number, randomUnit);
         ByteSizeValue secondByteValue = new ByteSizeValue(number, ByteSizeUnit.PB);
@@ -176,10 +176,25 @@ public class ByteSizeValueTests extends ESTestCase {
         assertTrue(secondByteValue.compareTo(firstByteValue) > 0);
     }
 
-    public void testEdgeCompare() {
-        ByteSizeValue maxLongValuePB = new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.PB);
-        ByteSizeValue maxLongValueB = new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES);
-        assertTrue(maxLongValuePB.compareTo(maxLongValueB) > 0);
+    public void testOutOfRange() {
+        // Make sure a value of > Long.MAX_VALUE bytes throws an exception
+        ByteSizeUnit unit = randomValueOtherThan(ByteSizeUnit.BYTES, () -> randomFrom(ByteSizeUnit.values()));
+        long size = (long) randomDouble() * unit.toBytes(1) + (Long.MAX_VALUE - unit.toBytes(1));
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new ByteSizeValue(size, unit));
+        assertEquals("Values greater than " + Long.MAX_VALUE + " bytes are not supported: " + size + unit.getSuffix(),
+                exception.getMessage());
+
+        // Make sure for units other than BYTES a size of -1 throws an exception
+        ByteSizeUnit unit2 = randomValueOtherThan(ByteSizeUnit.BYTES, () -> randomFrom(ByteSizeUnit.values()));
+        long size2 = -1L;
+        exception = expectThrows(IllegalArgumentException.class, () -> new ByteSizeValue(size2, unit2));
+        assertEquals("Values less than -1 bytes are not supported: " + size2 + unit2.getSuffix(), exception.getMessage());
+
+        // Make sure for any unit a size < -1 throws an exception
+        ByteSizeUnit unit3 = randomFrom(ByteSizeUnit.values());
+        long size3 = -1L * randomNonNegativeLong() - 1L;
+        exception = expectThrows(IllegalArgumentException.class, () -> new ByteSizeValue(size3, unit3));
+        assertEquals("Values less than -1 bytes are not supported: " + size3 + unit3.getSuffix(), exception.getMessage());
     }
 
     public void testConversionHashCode() {
@@ -188,14 +203,114 @@ public class ByteSizeValueTests extends ESTestCase {
         assertEquals(firstValue.hashCode(), secondValue.hashCode());
     }
 
-    public void testSerialization() throws IOException {
-        ByteSizeValue byteSizeValue = new ByteSizeValue(randomNonNegativeLong(), randomFrom(ByteSizeUnit.values()));
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            byteSizeValue.writeTo(out);
-            try (StreamInput in = out.bytes().streamInput()) {
-                ByteSizeValue deserializedByteSizeValue = new ByteSizeValue(in);
-                assertEquals(byteSizeValue.getBytes(), deserializedByteSizeValue.getBytes());
+    @Override
+    protected ByteSizeValue createTestInstance() {
+        if (randomBoolean()) {
+            ByteSizeUnit unit = randomFrom(ByteSizeUnit.values());
+            long size = randomNonNegativeLong() / unit.toBytes(1);
+            if (size >= Long.MAX_VALUE / unit.toBytes(1)) {
+                throw new AssertionError();
+            }
+            return new ByteSizeValue(size, unit);
+        } else {
+            return new ByteSizeValue(randomNonNegativeLong());
+        }
+    }
+
+    @Override
+    protected Reader instanceReader() {
+        return ByteSizeValue::new;
+    }
+
+    @Override
+    protected ByteSizeValue mutateInstance(ByteSizeValue instance) throws IOException {
+        long size = instance.getSize();
+        ByteSizeUnit unit = instance.getUnit();
+        switch (between(0, 1)) {
+        case 0:
+            long unitBytes = unit.toBytes(1);
+            size = randomValueOtherThan(size, () -> randomNonNegativeLong() / unitBytes);
+            break;
+        case 1:
+            unit = randomValueOtherThan(unit, () -> randomFrom(ByteSizeUnit.values()));
+            long newUnitBytes = unit.toBytes(1);
+            if (size >= Long.MAX_VALUE / newUnitBytes) {
+                size = randomValueOtherThan(size, () -> randomNonNegativeLong() / newUnitBytes);
+            }
+            break;
+        default:
+            throw new AssertionError("Invalid randomisation branch");
+        }
+        return new ByteSizeValue(size, unit);
+    }
+
+    public void testParse() {
+        for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
+            ByteSizeValue original = createTestInstance();
+            String serialised = original.getStringRep();
+            ByteSizeValue copy = ByteSizeValue.parseBytesSizeValue(serialised, "test");
+            assertEquals(original, copy);
+            assertEquals(serialised, copy.getStringRep());
+        }
+    }
+
+    public void testParseInvalidValue() {
+        ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class,
+                () -> ByteSizeValue.parseBytesSizeValue("-6mb", "test_setting"));
+        assertEquals("failed to parse setting [test_setting] with value [-6mb] as a size in bytes", exception.getMessage());
+        assertNotNull(exception.getCause());
+        assertEquals(IllegalArgumentException.class, exception.getCause().getClass());
+    }
+
+    public void testParseDefaultValue() {
+        ByteSizeValue defaultValue = createTestInstance();
+        assertEquals(defaultValue, ByteSizeValue.parseBytesSizeValue(null, defaultValue, "test"));
+    }
+
+    public void testParseSpecialValues() throws IOException {
+        ByteSizeValue instance = new ByteSizeValue(-1);
+        assertEquals(instance, ByteSizeValue.parseBytesSizeValue(instance.getStringRep(), null, "test"));
+        assertSerialization(instance);
+
+        instance = new ByteSizeValue(0);
+        assertEquals(instance, ByteSizeValue.parseBytesSizeValue(instance.getStringRep(), null, "test"));
+        assertSerialization(instance);
+    }
+
+    public void testParseInvalidNumber() throws IOException {
+        ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class,
+                () -> ByteSizeValue.parseBytesSizeValue("notANumber", "test"));
+        assertEquals("failed to parse setting [test] with value [notANumber] as a size in bytes: unit is missing or unrecognized",
+                exception.getMessage());
+
+        exception = expectThrows(ElasticsearchParseException.class, () -> ByteSizeValue.parseBytesSizeValue("notANumberMB", "test"));
+        assertEquals("failed to parse [notANumberMB]", exception.getMessage());
+    }
+
+    public void testParseFractionalNumber() throws IOException {
+        ByteSizeUnit unit = randomValueOtherThan(ByteSizeUnit.BYTES, () -> randomFrom(ByteSizeUnit.values()));
+        String fractionalValue = "23.5" + unit.getSuffix();
+        ByteSizeValue instance = ByteSizeValue.parseBytesSizeValue(fractionalValue, "test");
+        assertEquals(fractionalValue, instance.toString());
+        assertWarnings("Fractional bytes values are deprecated. Use non-fractional bytes values instead: [" + fractionalValue
+                + "] found for setting [test]");
+    }
+
+    public void testGetBytesAsInt() {
+        for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
+            ByteSizeValue instance = new ByteSizeValue(randomIntBetween(1, 1000), randomFrom(ByteSizeUnit.values()));
+            long bytesValue = instance.getBytes();
+            if (bytesValue > Integer.MAX_VALUE) {
+                IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> instance.bytesAsInt());
+                assertEquals("size [" + instance.toString() + "] is bigger than max int", exception.getMessage());
+            } else {
+                assertEquals((int) bytesValue, instance.bytesAsInt());
             }
         }
     }
+
+    public void testOldSerialisation() throws IOException {
+        ByteSizeValue original = createTestInstance();
+        assertSerialization(original, randomFrom(Version.V_5_6_4, Version.V_5_6_5, Version.V_6_0_0, Version.V_6_0_1, Version.V_6_1_0));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
index 1149111f613..c5a8aa5bdc2 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
@@ -408,19 +408,21 @@ public class IndexSettingsTests extends ESTestCase {
 
     public void testTranslogFlushSizeThreshold() {
         ByteSizeValue translogFlushThresholdSize = new ByteSizeValue(Math.abs(randomInt()));
-        ByteSizeValue actualValue = ByteSizeValue.parseBytesSizeValue(translogFlushThresholdSize.toString(),
+        ByteSizeValue actualValue = ByteSizeValue.parseBytesSizeValue(translogFlushThresholdSize.getBytes() + "B",
             IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey());
         IndexMetaData metaData = newIndexMeta("index", Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-            .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), translogFlushThresholdSize.toString())
+                .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), translogFlushThresholdSize.getBytes() + "B")
             .build());
         IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
         assertEquals(actualValue, settings.getFlushThresholdSize());
         ByteSizeValue newTranslogFlushThresholdSize = new ByteSizeValue(Math.abs(randomInt()));
-        ByteSizeValue actualNewTranslogFlushThresholdSize = ByteSizeValue.parseBytesSizeValue(newTranslogFlushThresholdSize.toString(),
+        ByteSizeValue actualNewTranslogFlushThresholdSize = ByteSizeValue.parseBytesSizeValue(
+                newTranslogFlushThresholdSize.getBytes() + "B",
             IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey());
         settings.updateIndexMetaData(newIndexMeta("index", Settings.builder()
-            .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), newTranslogFlushThresholdSize.toString()).build()));
+                .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), newTranslogFlushThresholdSize.getBytes() + "B")
+                .build()));
         assertEquals(actualNewTranslogFlushThresholdSize, settings.getFlushThresholdSize());
     }
 
@@ -428,20 +430,20 @@ public class IndexSettingsTests extends ESTestCase {
         final ByteSizeValue size = new ByteSizeValue(Math.abs(randomInt()));
         final String key = IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING.getKey();
         final ByteSizeValue actualValue =
-                ByteSizeValue.parseBytesSizeValue(size.toString(), key);
+                ByteSizeValue.parseBytesSizeValue(size.getBytes() + "B", key);
         final IndexMetaData metaData =
                 newIndexMeta(
                         "index",
                         Settings.builder()
                                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                                .put(key, size.toString())
+                                .put(key, size.getBytes() + "B")
                                 .build());
         final IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
         assertEquals(actualValue, settings.getGenerationThresholdSize());
         final ByteSizeValue newSize = new ByteSizeValue(Math.abs(randomInt()));
-        final ByteSizeValue actual = ByteSizeValue.parseBytesSizeValue(newSize.toString(), key);
+        final ByteSizeValue actual = ByteSizeValue.parseBytesSizeValue(newSize.getBytes() + "B", key);
         settings.updateIndexMetaData(
-                newIndexMeta("index", Settings.builder().put(key, newSize.toString()).build()));
+                newIndexMeta("index", Settings.builder().put(key, newSize.getBytes() + "B").build()));
         assertEquals(actual, settings.getGenerationThresholdSize());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index c079ebea840..c9cc771370e 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -35,8 +35,8 @@ import org.elasticsearch.index.shard.IndexShardTestCase;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.Scheduler.Cancellable;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -250,7 +250,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
         Exception e = expectThrows(IllegalArgumentException.class,
                                    () -> new MockController(Settings.builder()
                                                             .put("indices.memory.min_index_buffer_size", "-6mb").build()));
-        assertEquals("Failed to parse value [-6mb] for setting [indices.memory.min_index_buffer_size] must be >= 0b", e.getMessage());
+        assertEquals("failed to parse setting [indices.memory.min_index_buffer_size] with value [-6mb] as a size in bytes", e.getMessage());
 
     }
 
@@ -274,7 +274,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
         Exception e = expectThrows(IllegalArgumentException.class,
                                    () -> new MockController(Settings.builder()
                                                             .put("indices.memory.max_index_buffer_size", "-6mb").build()));
-        assertEquals("Failed to parse value [-6mb] for setting [indices.memory.max_index_buffer_size] must be >= -1b", e.getMessage());
+        assertEquals("failed to parse setting [indices.memory.max_index_buffer_size] with value [-6mb] as a size in bytes", e.getMessage());
 
     }
 
diff --git a/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java b/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java
index 0f05e623589..8c0820fc5b5 100644
--- a/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java
@@ -34,8 +34,10 @@ public class OsStatsTests extends ESTestCase {
             loadAverages[i] = randomDouble();
         }
         OsStats.Cpu cpu = new OsStats.Cpu(randomShort(), loadAverages);
-        OsStats.Mem mem = new OsStats.Mem(randomLong(), randomLong());
-        OsStats.Swap swap = new OsStats.Swap(randomLong(), randomLong());
+        long memTotal = randomNonNegativeLong();
+        OsStats.Mem mem = new OsStats.Mem(memTotal, randomLongBetween(0, memTotal));
+        long swapTotal = randomNonNegativeLong();
+        OsStats.Swap swap = new OsStats.Swap(swapTotal, randomLongBetween(0, swapTotal));
         OsStats.Cgroup cgroup = new OsStats.Cgroup(
             randomAlphaOfLength(8),
             randomNonNegativeLong(),
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
index c319d08ebf6..93508f11c09 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
@@ -19,10 +19,9 @@
 
 package org.elasticsearch.repositories.s3;
 
-import java.io.IOException;
-
 import com.amazonaws.services.s3.AbstractAmazonS3;
 import com.amazonaws.services.s3.AmazonS3;
+
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.settings.Settings;
@@ -33,6 +32,8 @@ import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
+import java.io.IOException;
+
 import static org.hamcrest.Matchers.containsString;
 
 public class S3RepositoryTests extends ESTestCase {
@@ -69,23 +70,23 @@ public class S3RepositoryTests extends ESTestCase {
         assertValidBuffer(5, 5);
         // buffer < 5mb should fail
         assertInvalidBuffer(4, 10, IllegalArgumentException.class,
-            "Failed to parse value [4mb] for setting [buffer_size] must be >= 5mb");
+                "Failed to parse value [4mb] for setting [buffer_size] must be >= 5mb");
         // chunk > 5tb should fail
         assertInvalidBuffer(5, 6000000, IllegalArgumentException.class,
-            "Failed to parse value [5.7tb] for setting [chunk_size] must be <= 5tb");
+                "Failed to parse value [6000000mb] for setting [chunk_size] must be <= 5tb");
     }
 
     private void assertValidBuffer(long bufferMB, long chunkMB) throws IOException {
         RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.builder()
-            .put(S3Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB))
-            .put(S3Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB)).build());
+                .put(S3Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB).getStringRep())
+                .put(S3Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB).getStringRep()).build());
         new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service());
     }
 
     private void assertInvalidBuffer(int bufferMB, int chunkMB, Class clazz, String msg) throws IOException {
         RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.builder()
-            .put(S3Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB))
-            .put(S3Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB)).build());
+                .put(S3Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB).getStringRep())
+                .put(S3Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB).getStringRep()).build());
 
         Exception e = expectThrows(clazz, () -> new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY,
             new DummyS3Service()));

From 10d2667498f0edc7addcb2f5f9dde564900ca199 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Thu, 14 Dec 2017 07:47:50 -0500
Subject: [PATCH 265/297] Clarify using supported LTS versions of the Java

This commit clarifies that we recommended using supported LTS versions
of Java as opposed to supporting a minimum version and any version above
that.

Relates #27795
---
 docs/Versions.asciidoc        |  1 +
 docs/reference/setup.asciidoc | 13 ++++++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index 7f6d952deca..2cc0616de6c 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -4,6 +4,7 @@
 :lucene_version_path:   7_2_0
 :branch:                master
 :jdk:                   1.8.0_131
+:jdk_major:             8
 
 //////////
 release-state can be: released | prerelease | unreleased
diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc
index 814abcd6527..b8516bdc6cb 100644
--- a/docs/reference/setup.asciidoc
+++ b/docs/reference/setup.asciidoc
@@ -16,7 +16,7 @@ running, including:
 == Supported platforms
 
 The matrix of officially supported operating systems and JVMs is available here:
-link:/support/matrix[Support Matrix].  Elasticsearch is tested on the listed
+link:/support/matrix[Support Matrix]. Elasticsearch is tested on the listed
 platforms, but it is possible that it will work on other platforms too.
 
 [float]
@@ -24,12 +24,15 @@ platforms, but it is possible that it will work on other platforms too.
 == Java (JVM) Version
 
 Elasticsearch is built using Java, and requires at least
-http://www.oracle.com/technetwork/java/javase/downloads/index.html[Java 8] in
-order to run. Only Oracle's Java and the OpenJDK are supported. The same JVM
+http://www.oracle.com/technetwork/java/javase/downloads/index.html[Java {jdk_major}]
+in order to run. Only Oracle's Java and the OpenJDK are supported. The same JVM
 version should be used on all Elasticsearch nodes and clients.
 
-We recommend installing Java version *{jdk} or later*. Elasticsearch will
-refuse to start if a known-bad version of Java is used.
+We recommend installing Java version *{jdk} or a later version in the Java
+{jdk_major} release series*. We recommend using a
+link:/support/matrix[supported]
+http://www.oracle.com/technetwork/java/eol-135779.html[LTS version of Java].
+Elasticsearch will refuse to start if a known-bad version of Java is used.
 
 The version of Java that Elasticsearch will use can be configured by setting
 the `JAVA_HOME` environment variable.

From 1b660821a21853b98a7a46683ad2e3442dd29dc9 Mon Sep 17 00:00:00 2001
From: Adrien Grand 
Date: Thu, 14 Dec 2017 17:47:53 +0100
Subject: [PATCH 266/297] Allow `_doc` as a type. (#27816)

Allowing `_doc` as a type will enable users to make the transition to 7.0
smoother since the index APIs will be `PUT index/_doc/id` and `POST index/_doc`.
This also moves most of the documentation to `_doc` as a type name.

Closes #27750
Closes #27751
---
 .../metadata/MetaDataMappingService.java      |   4 +-
 .../index/mapper/MapperService.java           |  45 ++++----
 .../cluster/metadata/MetaDataTests.java       |  44 ++++----
 .../discovery/ClusterDisruptionIT.java        |   6 +-
 .../gateway/MetaDataWriteDataNodesIT.java     |  20 ++--
 .../org/elasticsearch/get/GetActionIT.java    |  52 ++++-----
 .../index/mapper/AllFieldMapperTests.java     |   6 +-
 .../mapper/CopyToMapperIntegrationIT.java     |  12 +--
 .../index/mapper/CopyToMapperTests.java       |  32 +++---
 .../index/mapper/DateFieldMapperTests.java    |   8 +-
 .../index/mapper/DynamicMappingTests.java     |  12 +--
 .../ExternalValuesMapperIntegrationIT.java    |   6 +-
 .../mapper/FieldFilterMapperPluginTests.java  |  18 ++--
 .../index/mapper/MapperServiceTests.java      |  11 ++
 .../index/mapper/NestedObjectMapperTests.java |   4 +-
 .../index/query/MatchQueryBuilderTests.java   |   4 +-
 .../index/query/NestedQueryBuilderTests.java  |   2 +-
 .../query/QueryStringQueryBuilderTests.java   |   8 +-
 .../query/TermsSetQueryBuilderTests.java      |   2 +-
 .../index/query/TypeQueryBuilderTests.java    |   2 +-
 .../index/shard/IndexShardIT.java             |   6 +-
 .../termvectors/TermVectorsServiceTests.java  |   8 +-
 .../elasticsearch/indices/flush/FlushIT.java  |   2 +-
 .../mapping/SimpleGetFieldMappingsIT.java     |   6 +-
 .../indices/mapping/SimpleGetMappingsIT.java  |   2 +-
 .../mapping/UpdateMappingIntegrationIT.java   |  18 ++--
 .../indices/stats/IndexStatsIT.java           |  24 ++---
 .../template/SimpleIndexTemplateIT.java       |  12 +--
 .../SignificantTermsSignificanceScoreIT.java  |   8 +-
 .../search/query/QueryStringIT.java           |  28 ++---
 .../search/query/SearchQueryIT.java           |  26 ++---
 .../search/query/SimpleQueryStringIT.java     |  26 ++---
 .../MinThreadsSnapshotRestoreIT.java          |  10 +-
 .../SharedClusterSnapshotRestoreIT.java       | 102 +++++++++---------
 .../SharedSignificantTermsTestMethods.java    |   4 +-
 .../search/query/all-query-index.json         |   2 +-
 docs/build.gradle                             |  40 +++----
 .../adjacency-matrix-aggregation.asciidoc     |   4 +-
 .../bucket/children-aggregation.asciidoc      |   8 +-
 .../bucket/datehistogram-aggregation.asciidoc |   8 +-
 .../bucket/filters-aggregation.asciidoc       |   4 +-
 .../bucket/iprange-aggregation.asciidoc       |  10 +-
 .../matrix/stats-aggregation.asciidoc         |   6 +-
 .../metrics/cardinality-aggregation.asciidoc  |   4 +-
 .../metrics/percentile-aggregation.asciidoc   |  16 +--
 .../percentile-rank-aggregation.asciidoc      |  10 +-
 .../scripted-metric-aggregation.asciidoc      |   2 +-
 .../metrics/tophits-aggregation.asciidoc      |  14 +--
 docs/reference/aggregations/misc.asciidoc     |   8 +-
 docs/reference/analysis.asciidoc              |   2 +-
 .../analysis/analyzers/configuring.asciidoc   |   2 +-
 .../pattern-replace-charfilter.asciidoc       |   6 +-
 docs/reference/analysis/testing.asciidoc      |   2 +-
 .../tokenizers/edgengram-tokenizer.asciidoc   |   6 +-
 docs/reference/docs/bulk.asciidoc             |  30 +++---
 docs/reference/docs/delete-by-query.asciidoc  |   2 +-
 docs/reference/docs/delete.asciidoc           |  10 +-
 docs/reference/docs/get.asciidoc              |  34 +++---
 docs/reference/docs/index_.asciidoc           |  20 ++--
 docs/reference/docs/multi-get.asciidoc        |  64 ++---------
 .../reference/docs/multi-termvectors.asciidoc |  16 +--
 docs/reference/docs/refresh.asciidoc          |  10 +-
 docs/reference/docs/reindex.asciidoc          |  20 ++--
 docs/reference/docs/termvectors.asciidoc      |  26 ++---
 docs/reference/docs/update-by-query.asciidoc  |  10 +-
 docs/reference/docs/update.asciidoc           |  24 ++---
 docs/reference/getting-started.asciidoc       |  42 ++++----
 .../how-to/recipes/stemming.asciidoc          |  14 +--
 docs/reference/how-to/search-speed.asciidoc   |  10 +-
 .../index-modules/index-sorting.asciidoc      |   4 +-
 .../index-modules/similarity.asciidoc         |  18 ++--
 .../indices/get-field-mapping.asciidoc        |  18 ++--
 docs/reference/indices/get-mapping.asciidoc   |   6 +-
 docs/reference/indices/put-mapping.asciidoc   |  10 +-
 .../reference/indices/rollover-index.asciidoc |   2 +-
 docs/reference/ingest.asciidoc                |   2 +-
 docs/reference/ingest/ingest-node.asciidoc    |  32 +++---
 .../mapping/dynamic-mapping.asciidoc          |   4 +-
 .../mapping/dynamic/field-mapping.asciidoc    |  14 +--
 .../mapping/dynamic/templates.asciidoc        |  24 ++---
 .../mapping/fields/field-names-field.asciidoc |   6 +-
 .../mapping/fields/id-field.asciidoc          |   4 +-
 .../mapping/fields/index-field.asciidoc       |   4 +-
 .../mapping/fields/routing-field.asciidoc     |  10 +-
 .../mapping/fields/source-field.asciidoc      |   8 +-
 .../mapping/fields/type-field.asciidoc        |   4 +-
 .../mapping/fields/uid-field.asciidoc         |   6 +-
 .../mapping/params/analyzer.asciidoc          |  10 +-
 docs/reference/mapping/params/boost.asciidoc  |   2 +-
 docs/reference/mapping/params/coerce.asciidoc |  12 +--
 .../reference/mapping/params/copy-to.asciidoc |   4 +-
 .../mapping/params/doc-values.asciidoc        |   2 +-
 .../reference/mapping/params/dynamic.asciidoc |   6 +-
 .../params/eager-global-ordinals.asciidoc     |   4 +-
 .../reference/mapping/params/enabled.asciidoc |  14 +--
 .../mapping/params/fielddata.asciidoc         |   6 +-
 docs/reference/mapping/params/format.asciidoc |   2 +-
 .../mapping/params/ignore-above.asciidoc      |   6 +-
 .../mapping/params/ignore-malformed.asciidoc  |   8 +-
 .../mapping/params/index-options.asciidoc     |   4 +-
 .../mapping/params/multi-fields.asciidoc      |  12 +--
 .../mapping/params/normalizer.asciidoc        |  12 +--
 docs/reference/mapping/params/norms.asciidoc  |   2 +-
 .../mapping/params/null-value.asciidoc        |   6 +-
 .../params/position-increment-gap.asciidoc    |  12 +--
 .../mapping/params/properties.asciidoc        |   6 +-
 .../mapping/params/search-analyzer.asciidoc   |   4 +-
 .../mapping/params/similarity.asciidoc        |   2 +-
 docs/reference/mapping/params/store.asciidoc  |   4 +-
 .../mapping/params/term-vector.asciidoc       |   4 +-
 .../mapping/removal_of_types.asciidoc         |  22 ++--
 docs/reference/mapping/types/array.asciidoc   |   4 +-
 docs/reference/mapping/types/binary.asciidoc  |   4 +-
 docs/reference/mapping/types/boolean.asciidoc |   8 +-
 docs/reference/mapping/types/date.asciidoc    |  10 +-
 .../mapping/types/geo-point.asciidoc          |  10 +-
 docs/reference/mapping/types/ip.asciidoc      |   4 +-
 docs/reference/mapping/types/keyword.asciidoc |   2 +-
 docs/reference/mapping/types/nested.asciidoc  |   6 +-
 docs/reference/mapping/types/numeric.asciidoc |   2 +-
 docs/reference/mapping/types/object.asciidoc  |   4 +-
 .../mapping/types/parent-join.asciidoc        |  30 +++---
 .../mapping/types/percolator.asciidoc         |  18 ++--
 docs/reference/mapping/types/range.asciidoc   |   6 +-
 docs/reference/mapping/types/text.asciidoc    |   2 +-
 .../mapping/types/token-count.asciidoc        |   6 +-
 .../modules/cross-cluster-search.asciidoc     |  16 +--
 .../modules/scripting/fields.asciidoc         |  10 +-
 .../modules/scripting/using.asciidoc          |   2 +-
 .../query-dsl/geo-bounding-box-query.asciidoc |   4 +-
 .../query-dsl/geo-distance-query.asciidoc     |  14 +--
 .../query-dsl/geo-shape-query.asciidoc        |  10 +-
 docs/reference/query-dsl/ids-query.asciidoc   |   2 +-
 docs/reference/query-dsl/mlt-query.asciidoc   |   2 +-
 .../query-dsl/parent-id-query.asciidoc        |   6 +-
 .../query-dsl/percolate-query.asciidoc        |  26 ++---
 docs/reference/query-dsl/term-query.asciidoc  |  12 +--
 docs/reference/query-dsl/terms-query.asciidoc |   6 +-
 .../query-dsl/terms-set-query.asciidoc        |   8 +-
 docs/reference/query-dsl/type-query.asciidoc  |   2 +-
 docs/reference/search.asciidoc                |   2 +-
 docs/reference/search/count.asciidoc          |   6 +-
 docs/reference/search/explain.asciidoc        |   6 +-
 docs/reference/search/request-body.asciidoc   |   4 +-
 .../search/request/collapse.asciidoc          |   6 +-
 .../search/request/highlighting.asciidoc      |   8 +-
 .../search/request/inner-hits.asciidoc        |  34 +++---
 .../search/request/post-filter.asciidoc       |   4 +-
 docs/reference/search/request/scroll.asciidoc |   8 +-
 .../search/request/search-after.asciidoc      |   4 +-
 docs/reference/search/request/sort.asciidoc   |   6 +-
 docs/reference/search/search.asciidoc         |   4 +-
 .../suggesters/completion-suggest.asciidoc    |  12 +--
 .../suggesters/context-suggest.asciidoc       |  10 +-
 docs/reference/search/uri-request.asciidoc    |   4 +-
 docs/reference/search/validate.asciidoc       |  12 +--
 ...angeFieldQueryStringQueryBuilderTests.java |   2 +-
 .../join/query/HasChildQueryBuilderTests.java |   4 +-
 .../query/HasParentQueryBuilderTests.java     |   4 +-
 .../join/query/ParentIdQueryBuilderTests.java |   4 +-
 .../PercolateQueryBuilderTests.java           |   2 +-
 .../test/AbstractQueryTestCase.java           |   2 +-
 162 files changed, 884 insertions(+), 908 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
index 865b58c468a..12a56f00bd4 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
@@ -293,7 +293,9 @@ public class MetaDataMappingService extends AbstractComponent {
             }
             assert mappingType != null;
 
-            if (!MapperService.DEFAULT_MAPPING.equals(mappingType) && mappingType.charAt(0) == '_') {
+            if (MapperService.DEFAULT_MAPPING.equals(mappingType) == false
+                    && MapperService.SINGLE_MAPPING_NAME.equals(mappingType) == false
+                    && mappingType.charAt(0) == '_') {
                 throw new InvalidTypeNameException("Document mapping type name can't start with '_', found: [" + mappingType + "]");
             }
             MetaData.Builder builder = MetaData.builder(metaData);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index e34a762f527..51ebe9d980b 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -90,6 +90,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
     }
 
     public static final String DEFAULT_MAPPING = "_default_";
+    public static final String SINGLE_MAPPING_NAME = "_doc";
     public static final Setting INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING =
         Setting.longSetting("index.mapping.nested_fields.limit", 50L, 0, Property.Dynamic, Property.IndexScope);
     // maximum allowed number of nested json objects across all fields in a single document
@@ -338,6 +339,27 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         return internalMerge(defaultMapper, defaultMappingSource, documentMappers, reason, updateAllTypes);
     }
 
+    static void validateTypeName(String type) {
+        if (type.length() == 0) {
+            throw new InvalidTypeNameException("mapping type name is empty");
+        }
+        if (type.length() > 255) {
+            throw new InvalidTypeNameException("mapping type name [" + type + "] is too long; limit is length 255 but was [" + type.length() + "]");
+        }
+        if (type.charAt(0) == '_' && SINGLE_MAPPING_NAME.equals(type) == false) {
+            throw new InvalidTypeNameException("mapping type name [" + type + "] can't start with '_' unless it is called [" + SINGLE_MAPPING_NAME + "]");
+        }
+        if (type.contains("#")) {
+            throw new InvalidTypeNameException("mapping type name [" + type + "] should not include '#' in it");
+        }
+        if (type.contains(",")) {
+            throw new InvalidTypeNameException("mapping type name [" + type + "] should not include ',' in it");
+        }
+        if (type.charAt(0) == '.') {
+            throw new IllegalArgumentException("mapping type name [" + type + "] must not start with a '.'");
+        }
+    }
+
     private synchronized Map internalMerge(@Nullable DocumentMapper defaultMapper, @Nullable String defaultMappingSource,
                                                                    List documentMappers, MergeReason reason, boolean updateAllTypes) {
         boolean hasNested = this.hasNested;
@@ -361,27 +383,10 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
 
         for (DocumentMapper mapper : documentMappers) {
             // check naming
-            if (mapper.type().length() == 0) {
-                throw new InvalidTypeNameException("mapping type name is empty");
-            }
-            if (mapper.type().length() > 255) {
-                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] is too long; limit is length 255 but was [" + mapper.type().length() + "]");
-            }
-            if (mapper.type().charAt(0) == '_') {
-                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] can't start with '_'");
-            }
-            if (mapper.type().contains("#")) {
-                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] should not include '#' in it");
-            }
-            if (mapper.type().contains(",")) {
-                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] should not include ',' in it");
-            }
+            validateTypeName(mapper.type());
             if (mapper.type().equals(mapper.parentFieldMapper().type())) {
                 throw new IllegalArgumentException("The [_parent.type] option can't point to the same type");
             }
-            if (typeNameStartsWithIllegalDot(mapper)) {
-                throw new IllegalArgumentException("mapping type name [" + mapper.type() + "] must not start with a '.'");
-            }
 
             // compute the merged DocumentMapper
             DocumentMapper oldMapper = mappers.get(mapper.type());
@@ -519,10 +524,6 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         return true;
     }
 
-    private boolean typeNameStartsWithIllegalDot(DocumentMapper mapper) {
-        return mapper.type().startsWith(".");
-    }
-
     private boolean assertSerialization(DocumentMapper mapper) {
         // capture the source now, it may change due to concurrent parsing
         final CompressedXContent mappingSource = mapper.mappingSource();
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index acf6525f99d..2e670666c61 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -241,11 +241,11 @@ public class MetaDataTests extends ESTestCase {
                 .put(IndexMetaData.builder("index1")
                     .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                         .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
-                    .putMapping("doc", FIND_MAPPINGS_TEST_ITEM))
+                    .putMapping("_doc", FIND_MAPPINGS_TEST_ITEM))
                 .put(IndexMetaData.builder("index2")
                     .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                         .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
-                    .putMapping("doc", FIND_MAPPINGS_TEST_ITEM)).build();
+                    .putMapping("_doc", FIND_MAPPINGS_TEST_ITEM)).build();
 
         {
             ImmutableOpenMap> mappings = metaData.findMappings(Strings.EMPTY_ARRAY,
@@ -266,7 +266,7 @@ public class MetaDataTests extends ESTestCase {
         {
             ImmutableOpenMap> mappings = metaData.findMappings(
                     new String[]{"index1", "index2"},
-                    new String[]{randomBoolean() ? "doc" : "_all"}, MapperPlugin.NOOP_FIELD_FILTER);
+                    new String[]{randomBoolean() ? "_doc" : "_all"}, MapperPlugin.NOOP_FIELD_FILTER);
             assertEquals(2, mappings.size());
             assertIndexMappingsNotFiltered(mappings, "index1");
             assertIndexMappingsNotFiltered(mappings, "index2");
@@ -274,7 +274,7 @@ public class MetaDataTests extends ESTestCase {
     }
 
     public void testFindMappingsNoOpFilters() throws IOException {
-        MappingMetaData originalMappingMetaData = new MappingMetaData("doc",
+        MappingMetaData originalMappingMetaData = new MappingMetaData("_doc",
                 XContentHelper.convertToMap(JsonXContent.jsonXContent, FIND_MAPPINGS_TEST_ITEM, true));
 
         MetaData metaData = MetaData.builder()
@@ -287,28 +287,28 @@ public class MetaDataTests extends ESTestCase {
             ImmutableOpenMap> mappings = metaData.findMappings(new String[]{"index1"},
                     randomBoolean() ? Strings.EMPTY_ARRAY : new String[]{"_all"}, MapperPlugin.NOOP_FIELD_FILTER);
             ImmutableOpenMap index1 = mappings.get("index1");
-            MappingMetaData mappingMetaData = index1.get("doc");
+            MappingMetaData mappingMetaData = index1.get("_doc");
             assertSame(originalMappingMetaData, mappingMetaData);
         }
         {
             ImmutableOpenMap> mappings = metaData.findMappings(new String[]{"index1"},
                     randomBoolean() ? Strings.EMPTY_ARRAY : new String[]{"_all"}, index -> field -> randomBoolean());
             ImmutableOpenMap index1 = mappings.get("index1");
-            MappingMetaData mappingMetaData = index1.get("doc");
+            MappingMetaData mappingMetaData = index1.get("_doc");
             assertNotSame(originalMappingMetaData, mappingMetaData);
         }
         {
             ImmutableOpenMap> mappings = metaData.findMappings(new String[]{"index1"},
-                    new String[]{"doc"}, MapperPlugin.NOOP_FIELD_FILTER);
+                    new String[]{"_doc"}, MapperPlugin.NOOP_FIELD_FILTER);
             ImmutableOpenMap index1 = mappings.get("index1");
-            MappingMetaData mappingMetaData = index1.get("doc");
+            MappingMetaData mappingMetaData = index1.get("_doc");
             assertSame(originalMappingMetaData, mappingMetaData);
         }
         {
             ImmutableOpenMap> mappings = metaData.findMappings(new String[]{"index1"},
-                    new String[]{"doc"}, index -> field -> randomBoolean());
+                    new String[]{"_doc"}, index -> field -> randomBoolean());
             ImmutableOpenMap index1 = mappings.get("index1");
-            MappingMetaData mappingMetaData = index1.get("doc");
+            MappingMetaData mappingMetaData = index1.get("_doc");
             assertNotSame(originalMappingMetaData, mappingMetaData);
         }
     }
@@ -318,7 +318,7 @@ public class MetaDataTests extends ESTestCase {
         String mapping = FIND_MAPPINGS_TEST_ITEM;
         if (randomBoolean()) {
             Map stringObjectMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, FIND_MAPPINGS_TEST_ITEM, false);
-            Map doc = (Map)stringObjectMap.get("doc");
+            Map doc = (Map)stringObjectMap.get("_doc");
             try (XContentBuilder builder = JsonXContent.contentBuilder()) {
                 builder.map(doc);
                 mapping = builder.string();
@@ -329,20 +329,20 @@ public class MetaDataTests extends ESTestCase {
                 .put(IndexMetaData.builder("index1")
                         .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                         .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
-                .putMapping("doc", mapping))
+                .putMapping("_doc", mapping))
                 .put(IndexMetaData.builder("index2")
                         .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
-                        .putMapping("doc", mapping))
+                        .putMapping("_doc", mapping))
                 .put(IndexMetaData.builder("index3")
                         .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
-                        .putMapping("doc", mapping)).build();
+                        .putMapping("_doc", mapping)).build();
 
         {
             ImmutableOpenMap> mappings = metaData.findMappings(
                     new String[]{"index1", "index2", "index3"},
-                    new String[]{"doc"}, index -> {
+                    new String[]{"_doc"}, index -> {
                         if (index.equals("index1")) {
                             return field -> field.startsWith("name.") == false && field.startsWith("properties.key.") == false
                                     && field.equals("age") == false && field.equals("address.location") == false;
@@ -362,7 +362,7 @@ public class MetaDataTests extends ESTestCase {
             assertNotNull(index1Mappings);
 
             assertEquals(1, index1Mappings.size());
-            MappingMetaData docMapping = index1Mappings.get("doc");
+            MappingMetaData docMapping = index1Mappings.get("_doc");
             assertNotNull(docMapping);
 
             Map sourceAsMap = docMapping.getSourceAsMap();
@@ -406,13 +406,13 @@ public class MetaDataTests extends ESTestCase {
         {
             ImmutableOpenMap> mappings = metaData.findMappings(
                     new String[]{"index1", "index2" , "index3"},
-                    new String[]{"doc"}, index -> field -> (index.equals("index3") && field.endsWith("keyword")));
+                    new String[]{"_doc"}, index -> field -> (index.equals("index3") && field.endsWith("keyword")));
 
             assertIndexMappingsNoFields(mappings, "index1");
             assertIndexMappingsNoFields(mappings, "index2");
             ImmutableOpenMap index3 = mappings.get("index3");
             assertEquals(1, index3.size());
-            MappingMetaData mappingMetaData = index3.get("doc");
+            MappingMetaData mappingMetaData = index3.get("_doc");
             Map sourceAsMap = mappingMetaData.getSourceAsMap();
             assertEquals(3, sourceAsMap.size());
             assertTrue(sourceAsMap.containsKey("_routing"));
@@ -442,7 +442,7 @@ public class MetaDataTests extends ESTestCase {
         {
             ImmutableOpenMap> mappings = metaData.findMappings(
                     new String[]{"index1", "index2" , "index3"},
-                    new String[]{"doc"}, index -> field -> (index.equals("index2")));
+                    new String[]{"_doc"}, index -> field -> (index.equals("index2")));
 
             assertIndexMappingsNoFields(mappings, "index1");
             assertIndexMappingsNoFields(mappings, "index3");
@@ -456,7 +456,7 @@ public class MetaDataTests extends ESTestCase {
         ImmutableOpenMap indexMappings = mappings.get(index);
         assertNotNull(indexMappings);
         assertEquals(1, indexMappings.size());
-        MappingMetaData docMapping = indexMappings.get("doc");
+        MappingMetaData docMapping = indexMappings.get("_doc");
         assertNotNull(docMapping);
         Map sourceAsMap = docMapping.getSourceAsMap();
         assertEquals(3, sourceAsMap.size());
@@ -473,7 +473,7 @@ public class MetaDataTests extends ESTestCase {
         assertNotNull(indexMappings);
 
         assertEquals(1, indexMappings.size());
-        MappingMetaData docMapping = indexMappings.get("doc");
+        MappingMetaData docMapping = indexMappings.get("_doc");
         assertNotNull(docMapping);
 
         Map sourceAsMap = docMapping.getSourceAsMap();
@@ -540,7 +540,7 @@ public class MetaDataTests extends ESTestCase {
     }
 
     private static final String FIND_MAPPINGS_TEST_ITEM = "{\n" +
-            "  \"doc\": {\n" +
+            "  \"_doc\": {\n" +
             "      \"_routing\": {\n" +
             "        \"required\":true\n" +
             "      }," +
diff --git a/core/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/core/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
index 38c9bcb7245..8d21c630638 100644
--- a/core/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
@@ -380,7 +380,7 @@ public class ClusterDisruptionIT extends AbstractDisruptionTestCase {
         final String node_2 = internalCluster().startDataOnlyNode();
         List indexRequestBuilderList = new ArrayList<>();
         for (int i = 0; i < 100; i++) {
-            indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("doc")
+            indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("_doc")
                 .setSource("{\"int_field\":1}", XContentType.JSON));
         }
         indexRandom(true, indexRequestBuilderList);
@@ -398,7 +398,7 @@ public class ClusterDisruptionIT extends AbstractDisruptionTestCase {
 
         ensureStableCluster(2);
         assertAcked(prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 0)));
-        index("index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
+        index("index", "_doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
         ensureGreen();
 
         internalCluster().restartNode(masterNode, new InternalTestCluster.RestartCallback() {
@@ -409,7 +409,7 @@ public class ClusterDisruptionIT extends AbstractDisruptionTestCase {
         });
 
         ensureGreen("index");
-        assertTrue(client().prepareGet("index", "doc", "1").get().isExists());
+        assertTrue(client().prepareGet("index", "_doc", "1").get().isExists());
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
index fc4caa0bc09..68a6d239802 100644
--- a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
@@ -49,7 +49,7 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
         String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
         String dataNode = internalCluster().startDataOnlyNode(Settings.EMPTY);
         assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.number_of_replicas", 0)));
-        index("test", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
+        index("test", "_doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
         ensureGreen("test");
         assertIndexInMetaState(dataNode, "test");
         assertIndexInMetaState(masterNode, "test");
@@ -64,7 +64,7 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
 
         String index = "index";
         assertAcked(prepareCreate(index).setSettings(Settings.builder().put("index.number_of_replicas", 0).put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "_name", node1)));
-        index(index, "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
+        index(index, "_doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
         ensureGreen();
         assertIndexInMetaState(node1, index);
         Index resolveIndex = resolveIndex(index);
@@ -99,7 +99,7 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
         assertThat(clusterStateResponse.getState().getMetaData().index(index).getState().name(), equalTo(IndexMetaData.State.CLOSE.name()));
 
         // update the mapping. this should cause the new meta data to be written although index is closed
-        client().admin().indices().preparePutMapping(index).setType("doc").setSource(jsonBuilder().startObject()
+        client().admin().indices().preparePutMapping(index).setType("_doc").setSource(jsonBuilder().startObject()
                 .startObject("properties")
                 .startObject("integer_field")
                 .field("type", "integer")
@@ -107,12 +107,12 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
                 .endObject()
                 .endObject()).get();
 
-        GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).addTypes("doc").get();
-        assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get(index).get("doc").getSourceAsMap().get("properties"))).get("integer_field"));
+        GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).addTypes("_doc").get();
+        assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get(index).get("_doc").getSourceAsMap().get("properties"))).get("integer_field"));
 
         // make sure it was also written on red node although index is closed
         ImmutableOpenMap indicesMetaData = getIndicesMetaDataOnNode(dataNode);
-        assertNotNull(((LinkedHashMap) (indicesMetaData.get(index).getMappings().get("doc").getSourceAsMap().get("properties"))).get("integer_field"));
+        assertNotNull(((LinkedHashMap) (indicesMetaData.get(index).getMappings().get("_doc").getSourceAsMap().get("properties"))).get("integer_field"));
         assertThat(indicesMetaData.get(index).getState(), equalTo(IndexMetaData.State.CLOSE));
 
         /* Try the same and see if this also works if node was just restarted.
@@ -124,7 +124,7 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
          * what we write. This is why we explicitly test for it.
          */
         internalCluster().restartNode(dataNode, new RestartCallback());
-        client().admin().indices().preparePutMapping(index).setType("doc").setSource(jsonBuilder().startObject()
+        client().admin().indices().preparePutMapping(index).setType("_doc").setSource(jsonBuilder().startObject()
                 .startObject("properties")
                 .startObject("float_field")
                 .field("type", "float")
@@ -132,12 +132,12 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
                 .endObject()
                 .endObject()).get();
 
-        getMappingsResponse = client().admin().indices().prepareGetMappings(index).addTypes("doc").get();
-        assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get(index).get("doc").getSourceAsMap().get("properties"))).get("float_field"));
+        getMappingsResponse = client().admin().indices().prepareGetMappings(index).addTypes("_doc").get();
+        assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get(index).get("_doc").getSourceAsMap().get("properties"))).get("float_field"));
 
         // make sure it was also written on red node although index is closed
         indicesMetaData = getIndicesMetaDataOnNode(dataNode);
-        assertNotNull(((LinkedHashMap) (indicesMetaData.get(index).getMappings().get("doc").getSourceAsMap().get("properties"))).get("float_field"));
+        assertNotNull(((LinkedHashMap) (indicesMetaData.get(index).getMappings().get("_doc").getSourceAsMap().get("properties"))).get("float_field"));
         assertThat(indicesMetaData.get(index).getState(), equalTo(IndexMetaData.State.CLOSE));
 
         // finally check that meta data is also written of index opened again
diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
index c6135c8f228..d468d58212d 100644
--- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java
+++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
@@ -577,17 +577,17 @@ public class GetActionIT extends ESIntegTestCase {
 
     public void testGetFieldsMetaDataWithRouting() throws Exception {
         assertAcked(prepareCreate("test")
-            .addMapping("doc", "field1", "type=keyword,store=true")
+            .addMapping("_doc", "field1", "type=keyword,store=true")
             .addAlias(new Alias("alias"))
             .setSettings(Settings.builder().put("index.refresh_interval", -1).put("index.version.created", Version.V_5_6_0.id)));
             // multi types in 5.6
 
-        client().prepareIndex("test", "doc", "1")
+        client().prepareIndex("test", "_doc", "1")
             .setRouting("1")
             .setSource(jsonBuilder().startObject().field("field1", "value").endObject())
             .get();
 
-        GetResponse getResponse = client().prepareGet(indexOrAlias(), "doc", "1")
+        GetResponse getResponse = client().prepareGet(indexOrAlias(), "_doc", "1")
             .setRouting("1")
             .setStoredFields("field1")
             .get();
@@ -599,7 +599,7 @@ public class GetActionIT extends ESIntegTestCase {
 
         flush();
 
-        getResponse = client().prepareGet(indexOrAlias(), "doc", "1")
+        getResponse = client().prepareGet(indexOrAlias(), "_doc", "1")
             .setStoredFields("field1")
             .setRouting("1")
             .get();
@@ -778,7 +778,7 @@ public class GetActionIT extends ESIntegTestCase {
                 "    \"refresh_interval\": \"-1\"\n" +
                 "  },\n" +
                 "  \"mappings\": {\n" +
-                "    \"doc\": {\n" +
+                "    \"_doc\": {\n" +
                 "      \"properties\": {\n" +
                 "        \"suggest\": {\n" +
                 "          \"type\": \"completion\"\n" +
@@ -798,16 +798,16 @@ public class GetActionIT extends ESIntegTestCase {
                 "  }\n" +
                 "}";
 
-        index("test", "doc", "1", doc);
+        index("test", "_doc", "1", doc);
         String[] fieldsList = {"suggest"};
         // before refresh - document is only in translog
-        assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList);
+        assertGetFieldsAlwaysNull(indexOrAlias(), "_doc", "1", fieldsList);
         refresh();
         //after refresh - document is in translog and also indexed
-        assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList);
+        assertGetFieldsAlwaysNull(indexOrAlias(), "_doc", "1", fieldsList);
         flush();
         //after flush - document is in not anymore translog - only indexed
-        assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList);
+        assertGetFieldsAlwaysNull(indexOrAlias(), "_doc", "1", fieldsList);
     }
 
     public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException {
@@ -821,17 +821,17 @@ public class GetActionIT extends ESIntegTestCase {
                 .addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON));
         ensureGreen();
 
-        client().prepareIndex("test", "doc", "1").setRouting("routingValue").setId("1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("test", "_doc", "1").setRouting("routingValue").setId("1").setSource("{}", XContentType.JSON).get();
 
         String[] fieldsList = {"_routing"};
         // before refresh - document is only in translog
-        assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "routingValue");
+        assertGetFieldsAlwaysWorks(indexOrAlias(), "_doc", "1", fieldsList, "routingValue");
         refresh();
         //after refresh - document is in translog and also indexed
-        assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "routingValue");
+        assertGetFieldsAlwaysWorks(indexOrAlias(), "_doc", "1", fieldsList, "routingValue");
         flush();
         //after flush - document is in not anymore translog - only indexed
-        assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "routingValue");
+        assertGetFieldsAlwaysWorks(indexOrAlias(), "_doc", "1", fieldsList, "routingValue");
     }
 
     public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException {
@@ -847,41 +847,41 @@ public class GetActionIT extends ESIntegTestCase {
         String doc = "{\n" +
             "  \"text\": \"some text.\"\n" +
             "}\n";
-        client().prepareIndex("test", "doc").setId("1").setSource(doc, XContentType.JSON).setRouting("1").get();
+        client().prepareIndex("test", "_doc").setId("1").setSource(doc, XContentType.JSON).setRouting("1").get();
         String[] fieldsList = {"_routing"};
         // before refresh - document is only in translog
-        assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
+        assertGetFieldsAlwaysWorks(indexOrAlias(), "_doc", "1", fieldsList, "1");
         refresh();
         //after refresh - document is in translog and also indexed
-        assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
+        assertGetFieldsAlwaysWorks(indexOrAlias(), "_doc", "1", fieldsList, "1");
         flush();
         //after flush - document is in not anymore translog - only indexed
-        assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
+        assertGetFieldsAlwaysWorks(indexOrAlias(), "_doc", "1", fieldsList, "1");
     }
 
     public void testGeneratedStringFieldsUnstored() throws IOException {
         indexSingleDocumentWithStringFieldsGeneratedFromText(false, randomBoolean());
         String[] fieldsList = {"_field_names"};
         // before refresh - document is only in translog
-        assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList);
+        assertGetFieldsAlwaysNull(indexOrAlias(), "_doc", "1", fieldsList);
         refresh();
         //after refresh - document is in translog and also indexed
-        assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList);
+        assertGetFieldsAlwaysNull(indexOrAlias(), "_doc", "1", fieldsList);
         flush();
         //after flush - document is in not anymore translog - only indexed
-        assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList);
+        assertGetFieldsAlwaysNull(indexOrAlias(), "_doc", "1", fieldsList);
     }
 
     public void testGeneratedStringFieldsStored() throws IOException {
         indexSingleDocumentWithStringFieldsGeneratedFromText(true, randomBoolean());
         String[] fieldsList = {"text1", "text2"};
         String[] alwaysNotStoredFieldsList = {"_field_names"};
-        assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList);
-        assertGetFieldsNull(indexOrAlias(), "doc", "1", alwaysNotStoredFieldsList);
+        assertGetFieldsAlwaysWorks(indexOrAlias(), "_doc", "1", fieldsList);
+        assertGetFieldsNull(indexOrAlias(), "_doc", "1", alwaysNotStoredFieldsList);
         flush();
         //after flush - document is in not anymore translog - only indexed
-        assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList);
-        assertGetFieldsNull(indexOrAlias(), "doc", "1", alwaysNotStoredFieldsList);
+        assertGetFieldsAlwaysWorks(indexOrAlias(), "_doc", "1", fieldsList);
+        assertGetFieldsNull(indexOrAlias(), "_doc", "1", alwaysNotStoredFieldsList);
     }
 
     void indexSingleDocumentWithStringFieldsGeneratedFromText(boolean stored, boolean sourceEnabled) {
@@ -893,7 +893,7 @@ public class GetActionIT extends ESIntegTestCase {
                 "    \"refresh_interval\": \"-1\"\n" +
                 "  },\n" +
                 "  \"mappings\": {\n" +
-                "    \"doc\": {\n" +
+                "    \"_doc\": {\n" +
                 "      \"_source\" : {\"enabled\" : " + sourceEnabled + "}," +
                 "      \"properties\": {\n" +
                 "        \"text1\": {\n" +
@@ -915,7 +915,7 @@ public class GetActionIT extends ESIntegTestCase {
                 "  \"text1\": \"some text.\"\n," +
                 "  \"text2\": \"more text.\"\n" +
                 "}\n";
-        index("test", "doc", "1", doc);
+        index("test", "_doc", "1", doc);
     }
 
     private void assertGetFieldsAlwaysWorks(String index, String type, String docId, String[] fields) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java
index 426656984f1..4ccc8bc215f 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java
@@ -32,9 +32,9 @@ public class AllFieldMapperTests extends ESSingleNodeTestCase {
         IndexService indexService = createIndex("test", Settings.builder()
                 .put("index.analysis.analyzer.default_search.type", "custom")
                 .put("index.analysis.analyzer.default_search.tokenizer", "standard").build());
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("doc").endObject().endObject().string();
-        indexService.mapperService().merge("doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false);
-        assertEquals(mapping, indexService.mapperService().documentMapper("doc").mapping().toString());
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("_doc").endObject().endObject().string();
+        indexService.mapperService().merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false);
+        assertEquals(mapping, indexService.mapperService().documentMapper("_doc").mapping().toString());
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java
index eddc51348f9..637a25b24d6 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java
@@ -39,13 +39,13 @@ public class CopyToMapperIntegrationIT extends ESIntegTestCase {
     public void testDynamicTemplateCopyTo() throws Exception {
         assertAcked(
                 client().admin().indices().prepareCreate("test-idx")
-                        .addMapping("doc", createDynamicTemplateMapping())
+                        .addMapping("_doc", createDynamicTemplateMapping())
         );
 
         int recordCount = between(1, 200);
 
         for (int i = 0; i < recordCount * 2; i++) {
-            client().prepareIndex("test-idx", "doc", Integer.toString(i))
+            client().prepareIndex("test-idx", "_doc", Integer.toString(i))
                     .setSource("test_field", "test " + i, "even", i % 2 == 0)
                     .get();
         }
@@ -69,7 +69,7 @@ public class CopyToMapperIntegrationIT extends ESIntegTestCase {
     }
 
     public void testDynamicObjectCopyTo() throws Exception {
-        String mapping = jsonBuilder().startObject().startObject("doc").startObject("properties")
+        String mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties")
             .startObject("foo")
                 .field("type", "text")
                 .field("copy_to", "root.top.child")
@@ -77,9 +77,9 @@ public class CopyToMapperIntegrationIT extends ESIntegTestCase {
             .endObject().endObject().endObject().string();
         assertAcked(
             client().admin().indices().prepareCreate("test-idx")
-                .addMapping("doc", mapping, XContentType.JSON)
+                .addMapping("_doc", mapping, XContentType.JSON)
         );
-        client().prepareIndex("test-idx", "doc", "1")
+        client().prepareIndex("test-idx", "_doc", "1")
             .setSource("foo", "bar")
             .get();
         client().admin().indices().prepareRefresh("test-idx").execute().actionGet();
@@ -89,7 +89,7 @@ public class CopyToMapperIntegrationIT extends ESIntegTestCase {
     }
 
     private XContentBuilder createDynamicTemplateMapping() throws IOException {
-        return XContentFactory.jsonBuilder().startObject().startObject("doc")
+        return XContentFactory.jsonBuilder().startObject().startObject("_doc")
                 .startArray("dynamic_templates")
 
                 .startObject().startObject("template_raw")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
index 3fb3b94b229..a0b6a1458e2 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
@@ -419,7 +419,7 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
     public void testCopyToChildNested() throws Exception {
         IndexService indexService = createIndex("test");
         XContentBuilder rootToNestedMapping = jsonBuilder().startObject()
-                .startObject("doc")
+                .startObject("_doc")
                     .startObject("properties")
                         .startObject("source")
                             .field("type", "long")
@@ -437,12 +437,12 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
                 .endObject()
             .endObject();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> indexService.mapperService().merge("doc", new CompressedXContent(rootToNestedMapping.bytes()),
+                () -> indexService.mapperService().merge("_doc", new CompressedXContent(rootToNestedMapping.bytes()),
                         MergeReason.MAPPING_UPDATE, false));
         assertThat(e.getMessage(), Matchers.startsWith("Illegal combination of [copy_to] and [nested] mappings"));
 
         XContentBuilder nestedToNestedMapping = jsonBuilder().startObject()
-                .startObject("doc")
+                .startObject("_doc")
                     .startObject("properties")
                         .startObject("n1")
                             .field("type", "nested")
@@ -465,14 +465,14 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
                 .endObject()
             .endObject();
         e = expectThrows(IllegalArgumentException.class,
-                () -> indexService.mapperService().merge("doc", new CompressedXContent(nestedToNestedMapping.bytes()),
+                () -> indexService.mapperService().merge("_doc", new CompressedXContent(nestedToNestedMapping.bytes()),
                         MergeReason.MAPPING_UPDATE, false));
     }
 
     public void testCopyToSiblingNested() throws Exception {
         IndexService indexService = createIndex("test");
         XContentBuilder rootToNestedMapping = jsonBuilder().startObject()
-                .startObject("doc")
+                .startObject("_doc")
                     .startObject("properties")
                         .startObject("n1")
                             .field("type", "nested")
@@ -495,7 +495,7 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
                 .endObject()
             .endObject();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> indexService.mapperService().merge("doc", new CompressedXContent(rootToNestedMapping.bytes()),
+                () -> indexService.mapperService().merge("_doc", new CompressedXContent(rootToNestedMapping.bytes()),
                         MergeReason.MAPPING_UPDATE, false));
         assertThat(e.getMessage(), Matchers.startsWith("Illegal combination of [copy_to] and [nested] mappings"));
     }
@@ -503,7 +503,7 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
     public void testCopyToObject() throws Exception {
         IndexService indexService = createIndex("test");
         XContentBuilder rootToNestedMapping = jsonBuilder().startObject()
-                .startObject("doc")
+                .startObject("_doc")
                     .startObject("properties")
                         .startObject("source")
                             .field("type", "long")
@@ -516,7 +516,7 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
                 .endObject()
             .endObject();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> indexService.mapperService().merge("doc", new CompressedXContent(rootToNestedMapping.bytes()),
+                () -> indexService.mapperService().merge("_doc", new CompressedXContent(rootToNestedMapping.bytes()),
                         MergeReason.MAPPING_UPDATE, false));
         assertThat(e.getMessage(), Matchers.startsWith("Cannot copy to field [target] since it is mapped as an object"));
     }
@@ -569,7 +569,7 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
     }
 
     public void testCopyToMultiField() throws Exception {
-        String mapping = jsonBuilder().startObject().startObject("doc")
+        String mapping = jsonBuilder().startObject().startObject("_doc")
                 .startObject("properties")
                     .startObject("my_field")
                         .field("type", "keyword")
@@ -585,12 +585,12 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
 
         MapperService mapperService = createIndex("test").mapperService();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> mapperService.merge("doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean()));
+                () -> mapperService.merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean()));
         assertEquals("[copy_to] may not be used to copy to a multi-field: [my_field.bar]", e.getMessage());
     }
 
     public void testNestedCopyTo() throws Exception {
-        String mapping = jsonBuilder().startObject().startObject("doc")
+        String mapping = jsonBuilder().startObject().startObject("_doc")
                 .startObject("properties")
                     .startObject("n")
                         .field("type", "nested")
@@ -608,11 +608,11 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject().string();
 
         MapperService mapperService = createIndex("test").mapperService();
-        mapperService.merge("doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean()); // no exception
+        mapperService.merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean()); // no exception
     }
 
     public void testNestedCopyToMultiField() throws Exception {
-        String mapping = jsonBuilder().startObject().startObject("doc")
+        String mapping = jsonBuilder().startObject().startObject("_doc")
                 .startObject("properties")
                     .startObject("n")
                         .field("type", "nested")
@@ -633,12 +633,12 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
 
         MapperService mapperService = createIndex("test").mapperService();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> mapperService.merge("doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean()));
+                () -> mapperService.merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean()));
         assertEquals("[copy_to] may not be used to copy to a multi-field: [n.my_field.bar]", e.getMessage());
     }
 
     public void testCopyFromMultiField() throws Exception {
-        String mapping = jsonBuilder().startObject().startObject("doc")
+        String mapping = jsonBuilder().startObject().startObject("_doc")
                 .startObject("properties")
                     .startObject("my_field")
                         .field("type", "keyword")
@@ -654,7 +654,7 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
 
         MapperService mapperService = createIndex("test").mapperService();
         MapperParsingException e = expectThrows(MapperParsingException.class,
-                () -> mapperService.merge("doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean()));
+                () -> mapperService.merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean()));
         assertThat(e.getMessage(),
                 Matchers.containsString("copy_to in multi fields is not allowed. Found the copy_to in field [bar] " +
                         "which is within a multi field."));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index 7a4749bcb3c..5776e9d618e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -397,15 +397,15 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
     }
 
     public void testMergeText() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("doc")
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("_doc")
                 .startObject("properties").startObject("date").field("type", "date").endObject()
                 .endObject().endObject().endObject().string();
-        DocumentMapper mapper = indexService.mapperService().parse("doc", new CompressedXContent(mapping), false);
+        DocumentMapper mapper = indexService.mapperService().parse("_doc", new CompressedXContent(mapping), false);
 
-        String mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("doc")
+        String mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_doc")
                 .startObject("properties").startObject("date").field("type", "text").endObject()
                 .endObject().endObject().endObject().string();
-        DocumentMapper update = indexService.mapperService().parse("doc", new CompressedXContent(mappingUpdate), false);
+        DocumentMapper update = indexService.mapperService().parse("_doc", new CompressedXContent(mappingUpdate), false);
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> mapper.merge(update.mapping(), randomBoolean()));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
index 023d2249f2f..b227833f344 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
@@ -534,7 +534,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
     public void testMixTemplateMultiFieldAndMappingReuse() throws Exception {
         IndexService indexService = createIndex("test");
         XContentBuilder mappings1 = jsonBuilder().startObject()
-                .startObject("doc")
+                .startObject("_doc")
                     .startArray("dynamic_templates")
                         .startObject()
                             .startObject("template1")
@@ -551,21 +551,21 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
                         .endObject()
                     .endArray()
                 .endObject().endObject();
-        indexService.mapperService().merge("doc", new CompressedXContent(mappings1.bytes()),
+        indexService.mapperService().merge("_doc", new CompressedXContent(mappings1.bytes()),
             MapperService.MergeReason.MAPPING_UPDATE, false);
 
         XContentBuilder json = XContentFactory.jsonBuilder().startObject()
                     .field("field", "foo")
                 .endObject();
-        SourceToParse source = SourceToParse.source("test", "doc", "1", json.bytes(), json.contentType());
-        DocumentMapper mapper = indexService.mapperService().documentMapper("doc");
+        SourceToParse source = SourceToParse.source("test", "_doc", "1", json.bytes(), json.contentType());
+        DocumentMapper mapper = indexService.mapperService().documentMapper("_doc");
         assertNull(mapper.mappers().getMapper("field.raw"));
         ParsedDocument parsed = mapper.parse(source);
         assertNotNull(parsed.dynamicMappingsUpdate());
 
-        indexService.mapperService().merge("doc", new CompressedXContent(parsed.dynamicMappingsUpdate().toString()),
+        indexService.mapperService().merge("_doc", new CompressedXContent(parsed.dynamicMappingsUpdate().toString()),
             MapperService.MergeReason.MAPPING_UPDATE, false);
-        mapper = indexService.mapperService().documentMapper("doc");
+        mapper = indexService.mapperService().documentMapper("_doc");
         assertNotNull(mapper.mappers().getMapper("field.raw"));
         parsed = mapper.parse(source);
         assertNull(parsed.dynamicMappingsUpdate());
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
index 6dacbc9f64a..48f07b6063c 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
@@ -131,8 +131,8 @@ public class ExternalValuesMapperIntegrationIT extends ESIntegTestCase {
     }
 
     public void testExternalValuesWithMultifield() throws Exception {
-        prepareCreate("test-idx").addMapping("doc",
-                XContentFactory.jsonBuilder().startObject().startObject("doc").startObject("properties")
+        prepareCreate("test-idx").addMapping("_doc",
+                XContentFactory.jsonBuilder().startObject().startObject("_doc").startObject("properties")
                 .startObject("f")
                     .field("type", ExternalMapperPlugin.EXTERNAL_UPPER)
                     .startObject("fields")
@@ -150,7 +150,7 @@ public class ExternalValuesMapperIntegrationIT extends ESIntegTestCase {
                 .endObject()
                 .endObject().endObject().endObject()).execute().get();
 
-        index("test-idx", "doc", "1", "f", "This is my text");
+        index("test-idx", "_doc", "1", "f", "This is my text");
         refresh();
 
         SearchResponse response = client().prepareSearch("test-idx")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
index 86587be951f..af29edcef30 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java
@@ -59,7 +59,7 @@ public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase {
         assertAcked(client().admin().indices().prepareCreate("index1"));
         assertAcked(client().admin().indices().prepareCreate("filtered"));
         assertAcked(client().admin().indices().preparePutMapping("index1", "filtered")
-                .setType("doc").setSource(TEST_ITEM, XContentType.JSON));
+                .setType("_doc").setSource(TEST_ITEM, XContentType.JSON));
     }
 
     public void testGetMappings() {
@@ -83,7 +83,7 @@ public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase {
         //as the one coming from a filtered index with same mappings
         GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("filtered").get();
         ImmutableOpenMap filtered = getMappingsResponse.getMappings().get("filtered");
-        assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", filtered.get("doc").getSourceAsMap()));
+        assertAcked(client().admin().indices().prepareCreate("test").addMapping("_doc", filtered.get("_doc").getSourceAsMap()));
         GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings("test").setFields("*").get();
         assertEquals(1, response.mappings().size());
         assertFieldMappings(response.mappings().get("test"), FILTERED_FLAT_FIELDS);
@@ -98,7 +98,7 @@ public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase {
         //as the one coming from a filtered index with same mappings
         GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("filtered").get();
         ImmutableOpenMap filteredMapping = getMappingsResponse.getMappings().get("filtered");
-        assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", filteredMapping.get("doc").getSourceAsMap()));
+        assertAcked(client().admin().indices().prepareCreate("test").addMapping("_doc", filteredMapping.get("_doc").getSourceAsMap()));
         FieldCapabilitiesResponse test = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("test")).actionGet();
         assertFieldCaps(test, FILTERED_FLAT_FIELDS);
     }
@@ -120,7 +120,7 @@ public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase {
     private static void assertFieldMappings(Map> mappings,
                                             String[] expectedFields) {
         assertEquals(1, mappings.size());
-        Map fields = new HashMap<>(mappings.get("doc"));
+        Map fields = new HashMap<>(mappings.get("_doc"));
         Set builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
         for (String field : builtInMetaDataFields) {
             GetFieldMappingsResponse.FieldMappingMetaData fieldMappingMetaData = fields.remove(field);
@@ -138,12 +138,12 @@ public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase {
         assertNotFiltered(mappings.get("index1"));
         ImmutableOpenMap filtered = mappings.get("filtered");
         assertFiltered(filtered);
-        assertMappingsAreValid(filtered.get("doc").getSourceAsMap());
+        assertMappingsAreValid(filtered.get("_doc").getSourceAsMap());
     }
 
     private void assertMappingsAreValid(Map sourceAsMap) {
         //check that the returned filtered mappings are still valid mappings by submitting them and retrieving them back
-        assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", sourceAsMap));
+        assertAcked(client().admin().indices().prepareCreate("test").addMapping("_doc", sourceAsMap));
         GetMappingsResponse testMappingsResponse = client().admin().indices().prepareGetMappings("test").get();
         assertEquals(1, testMappingsResponse.getMappings().size());
         //the mappings are returned unfiltered for this index, yet they are the same as the previous ones that were returned filtered
@@ -153,7 +153,7 @@ public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase {
     @SuppressWarnings("unchecked")
     private static void assertFiltered(ImmutableOpenMap mappings) {
         assertEquals(1, mappings.size());
-        MappingMetaData mappingMetaData = mappings.get("doc");
+        MappingMetaData mappingMetaData = mappings.get("_doc");
         assertNotNull(mappingMetaData);
         Map sourceAsMap = mappingMetaData.getSourceAsMap();
         assertEquals(4, sourceAsMap.size());
@@ -200,7 +200,7 @@ public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase {
     @SuppressWarnings("unchecked")
     private static void assertNotFiltered(ImmutableOpenMap mappings) {
         assertEquals(1, mappings.size());
-        MappingMetaData mappingMetaData = mappings.get("doc");
+        MappingMetaData mappingMetaData = mappings.get("_doc");
         assertNotNull(mappingMetaData);
         Map sourceAsMap = mappingMetaData.getSourceAsMap();
         assertEquals(4, sourceAsMap.size());
@@ -255,7 +255,7 @@ public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase {
     };
 
     private static final String TEST_ITEM = "{\n" +
-            "  \"doc\": {\n" +
+            "  \"_doc\": {\n" +
             "      \"_meta\": {\n" +
             "        \"version\":0.19\n" +
             "      }," +
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
index 3ab9ba8406b..95183cc854a 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
+import org.elasticsearch.indices.InvalidTypeNameException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -101,6 +102,16 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
         assertEquals(new HashSet<>(Arrays.asList("type1", "type2")), mapperService.types());
     }
 
+    public void testTypeValidation() {
+        InvalidTypeNameException e = expectThrows(InvalidTypeNameException.class, () -> MapperService.validateTypeName("_type"));
+        assertEquals("mapping type name [_type] can't start with '_' unless it is called [_doc]", e.getMessage());
+
+        e = expectThrows(InvalidTypeNameException.class, () -> MapperService.validateTypeName("_document"));
+        assertEquals("mapping type name [_document] can't start with '_' unless it is called [_doc]", e.getMessage());
+
+        MapperService.validateTypeName("_doc"); // no exception
+    }
+
     public void testIndexIntoDefaultMapping() throws Throwable {
         // 1. test implicit index creation
         ExecutionException e = expectThrows(ExecutionException.class, () -> {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
index e17d58d4a14..8da4b302a6f 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
@@ -495,7 +495,7 @@ public class NestedObjectMapperTests extends ESSingleNodeTestCase {
     }
 
     public void testParentObjectMapperAreNested() throws Exception {
-        MapperService mapperService = createIndex("index1", Settings.EMPTY, "doc", jsonBuilder().startObject()
+        MapperService mapperService = createIndex("index1", Settings.EMPTY, "_doc", jsonBuilder().startObject()
                 .startObject("properties")
                     .startObject("comments")
                         .field("type", "nested")
@@ -509,7 +509,7 @@ public class NestedObjectMapperTests extends ESSingleNodeTestCase {
         ObjectMapper objectMapper = mapperService.getObjectMapper("comments.messages");
         assertTrue(objectMapper.parentObjectMapperAreNested(mapperService));
 
-        mapperService = createIndex("index2", Settings.EMPTY, "doc", jsonBuilder().startObject()
+        mapperService = createIndex("index2", Settings.EMPTY, "_doc", jsonBuilder().startObject()
             .startObject("properties")
                 .startObject("comments")
                     .field("type", "object")
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index 9d5ee3e7f76..8849c91ddb3 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -350,8 +350,8 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase 0);
         QueryShardContext context = createShardContext();
-        context.getMapperService().merge("doc",
+        context.getMapperService().merge("_doc",
             new CompressedXContent(
-                PutMappingRequest.buildFromSimplifiedDef("doc",
+                PutMappingRequest.buildFromSimplifiedDef("_doc",
                     "foo", "type=text",
                     "_field_names", "enabled=false").string()),
             MapperService.MergeReason.MAPPING_UPDATE, true);
@@ -843,9 +843,9 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase indexRequestBuilders = new ArrayList<>();
         for (int i = 0; i < data.length; i++) {
             String[] parts = data[i].split("\t");
-            indexRequestBuilders.add(client().prepareIndex("test", "doc", "" + i)
+            indexRequestBuilders.add(client().prepareIndex("test", "_doc", "" + i)
                     .setSource("class", parts[0], "text", parts[1]));
         }
         indexRandom(true, false, indexRequestBuilders);
diff --git a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
index ab8bcb539d6..7145f9db2db 100644
--- a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
@@ -71,9 +71,9 @@ public class QueryStringIT extends ESIntegTestCase {
 
     public void testBasicAllQuery() throws Exception {
         List reqs = new ArrayList<>();
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo bar baz"));
-        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f2", "Bar"));
-        reqs.add(client().prepareIndex("test", "doc", "3").setSource("f3", "foo bar baz"));
+        reqs.add(client().prepareIndex("test", "_doc", "1").setSource("f1", "foo bar baz"));
+        reqs.add(client().prepareIndex("test", "_doc", "2").setSource("f2", "Bar"));
+        reqs.add(client().prepareIndex("test", "_doc", "3").setSource("f3", "foo bar baz"));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
@@ -91,8 +91,8 @@ public class QueryStringIT extends ESIntegTestCase {
 
     public void testWithDate() throws Exception {
         List reqs = new ArrayList<>();
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo", "f_date", "2015/09/02"));
-        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar", "f_date", "2015/09/01"));
+        reqs.add(client().prepareIndex("test", "_doc", "1").setSource("f1", "foo", "f_date", "2015/09/02"));
+        reqs.add(client().prepareIndex("test", "_doc", "2").setSource("f1", "bar", "f_date", "2015/09/01"));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo bar")).get();
@@ -114,11 +114,11 @@ public class QueryStringIT extends ESIntegTestCase {
 
     public void testWithLotsOfTypes() throws Exception {
         List reqs = new ArrayList<>();
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo",
+        reqs.add(client().prepareIndex("test", "_doc", "1").setSource("f1", "foo",
                         "f_date", "2015/09/02",
                         "f_float", "1.7",
                         "f_ip", "127.0.0.1"));
-        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar",
+        reqs.add(client().prepareIndex("test", "_doc", "2").setSource("f1", "bar",
                         "f_date", "2015/09/01",
                         "f_float", "1.8",
                         "f_ip", "127.0.0.2"));
@@ -144,7 +144,7 @@ public class QueryStringIT extends ESIntegTestCase {
     public void testDocWithAllTypes() throws Exception {
         List reqs = new ArrayList<>();
         String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json");
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody, XContentType.JSON));
+        reqs.add(client().prepareIndex("test", "_doc", "1").setSource(docBody, XContentType.JSON));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
@@ -181,9 +181,9 @@ public class QueryStringIT extends ESIntegTestCase {
 
     public void testKeywordWithWhitespace() throws Exception {
         List reqs = new ArrayList<>();
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f2", "Foo Bar"));
-        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar"));
-        reqs.add(client().prepareIndex("test", "doc", "3").setSource("f1", "foo bar"));
+        reqs.add(client().prepareIndex("test", "_doc", "1").setSource("f2", "Foo Bar"));
+        reqs.add(client().prepareIndex("test", "_doc", "2").setSource("f1", "bar"));
+        reqs.add(client().prepareIndex("test", "_doc", "3").setSource("f1", "foo bar"));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
@@ -209,7 +209,7 @@ public class QueryStringIT extends ESIntegTestCase {
         ensureGreen("test_1");
 
         List reqs = new ArrayList<>();
-        reqs.add(client().prepareIndex("test_1", "doc", "1").setSource("f1", "foo", "f2", "eggplant"));
+        reqs.add(client().prepareIndex("test_1", "_doc", "1").setSource("f1", "foo", "f2", "eggplant"));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test_1").setQuery(
@@ -225,8 +225,8 @@ public class QueryStringIT extends ESIntegTestCase {
 
     public void testPhraseQueryOnFieldWithNoPositions() throws Exception {
         List reqs = new ArrayList<>();
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo bar", "f4", "eggplant parmesan"));
-        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "foo bar", "f4", "chicken parmesan"));
+        reqs.add(client().prepareIndex("test", "_doc", "1").setSource("f1", "foo bar", "f4", "eggplant parmesan"));
+        reqs.add(client().prepareIndex("test", "_doc", "2").setSource("f1", "foo bar", "f4", "chicken parmesan"));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test")
diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index a94f499d0ba..4b56d2bc9e1 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -1194,12 +1194,12 @@ public class SearchQueryIT extends ESIntegTestCase {
     public void testBasicQueryById() throws Exception {
         assertAcked(prepareCreate("test"));
 
-        client().prepareIndex("test", "doc", "1").setSource("field1", "value1").get();
-        client().prepareIndex("test", "doc", "2").setSource("field1", "value2").get();
-        client().prepareIndex("test", "doc", "3").setSource("field1", "value3").get();
+        client().prepareIndex("test", "_doc", "1").setSource("field1", "value1").get();
+        client().prepareIndex("test", "_doc", "2").setSource("field1", "value2").get();
+        client().prepareIndex("test", "_doc", "3").setSource("field1", "value3").get();
         refresh();
 
-        SearchResponse searchResponse = client().prepareSearch().setQuery(idsQuery("doc").addIds("1", "2")).get();
+        SearchResponse searchResponse = client().prepareSearch().setQuery(idsQuery("_doc").addIds("1", "2")).get();
         assertHitCount(searchResponse, 2L);
         assertThat(searchResponse.getHits().getHits().length, equalTo(2));
 
@@ -1215,7 +1215,7 @@ public class SearchQueryIT extends ESIntegTestCase {
         assertHitCount(searchResponse, 1L);
         assertThat(searchResponse.getHits().getHits().length, equalTo(1));
 
-        searchResponse = client().prepareSearch().setQuery(idsQuery("type1", "type2", "doc").addIds("1", "2", "3", "4")).get();
+        searchResponse = client().prepareSearch().setQuery(idsQuery("type1", "type2", "_doc").addIds("1", "2", "3", "4")).get();
         assertHitCount(searchResponse, 3L);
         assertThat(searchResponse.getHits().getHits().length, equalTo(3));
     }
@@ -1489,9 +1489,9 @@ public class SearchQueryIT extends ESIntegTestCase {
 
     public void testSimpleDFSQuery() throws IOException {
         assertAcked(prepareCreate("test")
-            .addMapping("doc", jsonBuilder()
+            .addMapping("_doc", jsonBuilder()
                 .startObject()
-                .startObject("doc")
+                .startObject("_doc")
                 .startObject("_routing")
                 .field("required", true)
                 .endObject()
@@ -1513,13 +1513,13 @@ public class SearchQueryIT extends ESIntegTestCase {
             );
 
 
-        client().prepareIndex("test", "doc", "1").setRouting("Y").setSource("online", false, "bs", "Y", "ts",
+        client().prepareIndex("test", "_doc", "1").setRouting("Y").setSource("online", false, "bs", "Y", "ts",
             System.currentTimeMillis() - 100, "type", "s").get();
-        client().prepareIndex("test", "doc", "2").setRouting("X").setSource("online", true, "bs", "X", "ts",
+        client().prepareIndex("test", "_doc", "2").setRouting("X").setSource("online", true, "bs", "X", "ts",
             System.currentTimeMillis() - 10000000, "type", "s").get();
-        client().prepareIndex("test", "doc", "3").setRouting(randomAlphaOfLength(2))
+        client().prepareIndex("test", "_doc", "3").setRouting(randomAlphaOfLength(2))
             .setSource("online", false, "ts", System.currentTimeMillis() - 100, "type", "bs").get();
-        client().prepareIndex("test", "doc", "4").setRouting(randomAlphaOfLength(2))
+        client().prepareIndex("test", "_doc", "4").setRouting(randomAlphaOfLength(2))
             .setSource("online", true, "ts", System.currentTimeMillis() - 123123, "type", "bs").get();
         refresh();
 
@@ -1666,8 +1666,8 @@ public class SearchQueryIT extends ESIntegTestCase {
     public void testQueryStringWithSlopAndFields() {
         assertAcked(prepareCreate("test"));
 
-        client().prepareIndex("test", "doc", "1").setSource("desc", "one two three", "type", "customer").get();
-        client().prepareIndex("test", "doc", "2").setSource("desc", "one two three", "type", "product").get();
+        client().prepareIndex("test", "_doc", "1").setSource("desc", "one two three", "type", "customer").get();
+        client().prepareIndex("test", "_doc", "2").setSource("desc", "one two three", "type", "product").get();
         refresh();
         {
             SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.queryStringQuery("\"one two\"").defaultField("desc")).get();
diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
index bd4bf0624fe..cf2c3463cf9 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
@@ -292,8 +292,8 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
     // Issue #7967
     public void testLenientFlagBeingTooLenient() throws Exception {
         indexRandom(true,
-                client().prepareIndex("test", "doc", "1").setSource("num", 1, "body", "foo bar baz"),
-                client().prepareIndex("test", "doc", "2").setSource("num", 2, "body", "eggplant spaghetti lasagna"));
+                client().prepareIndex("test", "_doc", "1").setSource("num", 1, "body", "foo bar baz"),
+                client().prepareIndex("test", "_doc", "2").setSource("num", 2, "body", "eggplant spaghetti lasagna"));
 
         BoolQueryBuilder q = boolQuery().should(simpleQueryStringQuery("bar").field("num").field("body").lenient(true));
         SearchResponse resp = client().prepareSearch("test").setQuery(q).get();
@@ -386,9 +386,9 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
         ensureGreen("test");
 
         List reqs = new ArrayList<>();
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo bar baz"));
-        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f2", "Bar"));
-        reqs.add(client().prepareIndex("test", "doc", "3").setSource("f3", "foo bar baz"));
+        reqs.add(client().prepareIndex("test", "_doc", "1").setSource("f1", "foo bar baz"));
+        reqs.add(client().prepareIndex("test", "_doc", "2").setSource("f2", "Bar"));
+        reqs.add(client().prepareIndex("test", "_doc", "3").setSource("f3", "foo bar baz"));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
@@ -410,8 +410,8 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
         ensureGreen("test");
 
         List reqs = new ArrayList<>();
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo", "f_date", "2015/09/02"));
-        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar", "f_date", "2015/09/01"));
+        reqs.add(client().prepareIndex("test", "_doc", "1").setSource("f1", "foo", "f_date", "2015/09/02"));
+        reqs.add(client().prepareIndex("test", "_doc", "2").setSource("f1", "bar", "f_date", "2015/09/01"));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")).get();
@@ -437,11 +437,11 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
         ensureGreen("test");
 
         List reqs = new ArrayList<>();
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f1", "foo",
+        reqs.add(client().prepareIndex("test", "_doc", "1").setSource("f1", "foo",
                         "f_date", "2015/09/02",
                         "f_float", "1.7",
                         "f_ip", "127.0.0.1"));
-        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar",
+        reqs.add(client().prepareIndex("test", "_doc", "2").setSource("f1", "bar",
                         "f_date", "2015/09/01",
                         "f_float", "1.8",
                         "f_ip", "127.0.0.2"));
@@ -471,7 +471,7 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
 
         List reqs = new ArrayList<>();
         String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json");
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody, XContentType.JSON));
+        reqs.add(client().prepareIndex("test", "_doc", "1").setSource(docBody, XContentType.JSON));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
@@ -516,9 +516,9 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
         ensureGreen("test");
 
         List reqs = new ArrayList<>();
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource("f2", "Foo Bar"));
-        reqs.add(client().prepareIndex("test", "doc", "2").setSource("f1", "bar"));
-        reqs.add(client().prepareIndex("test", "doc", "3").setSource("f1", "foo bar"));
+        reqs.add(client().prepareIndex("test", "_doc", "1").setSource("f2", "Foo Bar"));
+        reqs.add(client().prepareIndex("test", "_doc", "2").setSource("f1", "bar"));
+        reqs.add(client().prepareIndex("test", "_doc", "3").setSource("f1", "foo bar"));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
diff --git a/core/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java
index 29657c5fb8b..6279ed28903 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java
@@ -66,7 +66,7 @@ public class MinThreadsSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
         final String index = "test-idx1";
         assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
         for (int i = 0; i < 10; i++) {
-            index(index, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(index, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         final String snapshot1 = "test-snap1";
@@ -74,7 +74,7 @@ public class MinThreadsSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
         final String index2 = "test-idx2";
         assertAcked(prepareCreate(index2, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
         for (int i = 0; i < 10; i++) {
-            index(index2, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(index2, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         final String snapshot2 = "test-snap2";
@@ -120,7 +120,7 @@ public class MinThreadsSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
         final String index = "test-idx";
         assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
         for (int i = 0; i < 10; i++) {
-            index(index, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(index, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         final String snapshot1 = "test-snap1";
@@ -166,7 +166,7 @@ public class MinThreadsSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
         final String index = "test-idx";
         assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
         for (int i = 0; i < 10; i++) {
-            index(index, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(index, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         final String snapshot1 = "test-snap1";
@@ -174,7 +174,7 @@ public class MinThreadsSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
         final String index2 = "test-idx2";
         assertAcked(prepareCreate(index2, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
         for (int i = 0; i < 10; i++) {
-            index(index2, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(index2, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         final String snapshot2 = "test-snap2";
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 3f9c80f3ffa..45ec0746a9b 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -173,9 +173,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
-            index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
-            index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
+            index("test-idx-1", "_doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx-2", "_doc", Integer.toString(i), "foo", "baz" + i);
+            index("test-idx-3", "_doc", Integer.toString(i), "foo", "baz" + i);
         }
         refresh();
         assertHitCount(client.prepareSearch("test-idx-1").setSize(0).get(), 100L);
@@ -225,13 +225,13 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> delete some data");
         for (int i = 0; i < 50; i++) {
-            client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
+            client.prepareDelete("test-idx-1", "_doc", Integer.toString(i)).get();
         }
         for (int i = 50; i < 100; i++) {
-            client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get();
+            client.prepareDelete("test-idx-2", "_doc", Integer.toString(i)).get();
         }
         for (int i = 0; i < 100; i += 2) {
-            client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get();
+            client.prepareDelete("test-idx-3", "_doc", Integer.toString(i)).get();
         }
         assertAllSuccessful(refresh());
         assertHitCount(client.prepareSearch("test-idx-1").setSize(0).get(), 50L);
@@ -553,9 +553,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
             assertThat(client.admin().indices()
                 .preparePutTemplate("test-template")
                 .setPatterns(Collections.singletonList("te*"))
-                .addMapping("doc", XContentFactory.jsonBuilder()
+                .addMapping("_doc", XContentFactory.jsonBuilder()
                     .startObject()
-                        .startObject("doc")
+                        .startObject("_doc")
                             .startObject("properties")
                                 .startObject("field1")
                                     .field("type", "text")
@@ -665,7 +665,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -723,7 +723,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -772,7 +772,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -836,7 +836,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -895,7 +895,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -1003,7 +1003,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         // index some documents
         final int nbDocs = scaledRandomIntBetween(10, 100);
         for (int i = 0; i < nbDocs; i++) {
-            index(indexName, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         flushAndRefresh(indexName);
         assertThat(client().prepareSearch(indexName).setSize(0).get().getHits().getTotalHits(), equalTo((long) nbDocs));
@@ -1095,7 +1095,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -1184,7 +1184,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         logger.info("--> creating {} snapshots ", numberOfSnapshots);
         for (int i = 0; i < numberOfSnapshots; i++) {
             for (int j = 0; j < 10; j++) {
-                index("test-idx", "doc", Integer.toString(i * 10 + j), "foo", "bar" + i * 10 + j);
+                index("test-idx", "_doc", Integer.toString(i * 10 + j), "foo", "bar" + i * 10 + j);
             }
             refresh();
             logger.info("--> snapshot {}", i);
@@ -1237,8 +1237,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         createIndex("test-idx-1", "test-idx-2");
         logger.info("--> indexing some data");
         indexRandom(true,
-                client().prepareIndex("test-idx-1", "doc").setSource("foo", "bar"),
-                client().prepareIndex("test-idx-2", "doc").setSource("foo", "bar"));
+                client().prepareIndex("test-idx-1", "_doc").setSource("foo", "bar"),
+                client().prepareIndex("test-idx-2", "_doc").setSource("foo", "bar"));
 
         logger.info("--> creating snapshot");
         CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get();
@@ -1274,8 +1274,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         createIndex("test-idx-1", "test-idx-2");
         logger.info("--> indexing some data");
         indexRandom(true,
-                client().prepareIndex("test-idx-1", "doc").setSource("foo", "bar"),
-                client().prepareIndex("test-idx-2", "doc").setSource("foo", "bar"));
+                client().prepareIndex("test-idx-1", "_doc").setSource("foo", "bar"),
+                client().prepareIndex("test-idx-2", "_doc").setSource("foo", "bar"));
 
         logger.info("--> creating snapshot");
         CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get();
@@ -1307,8 +1307,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         createIndex("test-idx-1", "test-idx-2");
         logger.info("--> indexing some data");
         indexRandom(true,
-                client().prepareIndex("test-idx-1", "doc").setSource("foo", "bar"),
-                client().prepareIndex("test-idx-2", "doc").setSource("foo", "bar"));
+                client().prepareIndex("test-idx-1", "_doc").setSource("foo", "bar"),
+                client().prepareIndex("test-idx-2", "_doc").setSource("foo", "bar"));
 
         logger.info("--> creating snapshot");
         CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get();
@@ -1341,8 +1341,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         createIndex("test-idx-1", "test-idx-2");
         logger.info("--> indexing some data");
         indexRandom(true,
-            client().prepareIndex("test-idx-1", "doc").setSource("foo", "bar"),
-            client().prepareIndex("test-idx-2", "doc").setSource("foo", "bar"));
+            client().prepareIndex("test-idx-1", "_doc").setSource("foo", "bar"),
+            client().prepareIndex("test-idx-2", "_doc").setSource("foo", "bar"));
 
         logger.info("--> creating snapshot");
         client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1")
@@ -1435,8 +1435,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
-            index("test-idx-2", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx-1", "_doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx-2", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -1551,7 +1551,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -1614,7 +1614,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -1695,7 +1695,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
 
@@ -1754,7 +1754,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -1810,7 +1810,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -1923,7 +1923,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -1963,7 +1963,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         final int numdocs = randomIntBetween(10, 100);
         IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs];
         for (int i = 0; i < builders.length; i++) {
-            builders[i] = client().prepareIndex("test", "doc", Integer.toString(i)).setSource("foo", "bar" + i);
+            builders[i] = client().prepareIndex("test", "_doc", Integer.toString(i)).setSource("foo", "bar" + i);
         }
         indexRandom(true, builders);
         flushAndRefresh();
@@ -1993,7 +1993,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
             }
         }
 
-        client().prepareDelete("test", "doc", "1").get();
+        client().prepareDelete("test", "_doc", "1").get();
         CreateSnapshotResponse createSnapshotResponseThird = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-2").setWaitForCompletion(true).setIndices("test").get();
         assertThat(createSnapshotResponseThird.getSnapshotInfo().successfulShards(), greaterThan(0));
         assertThat(createSnapshotResponseThird.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponseThird.getSnapshotInfo().totalShards()));
@@ -2241,9 +2241,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
-            index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
-            index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
+            index("test-idx-1", "_doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx-2", "_doc", Integer.toString(i), "foo", "baz" + i);
+            index("test-idx-3", "_doc", Integer.toString(i), "foo", "baz" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -2329,8 +2329,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
-            index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
+            index("test-idx-1", "_doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx-2", "_doc", Integer.toString(i), "foo", "baz" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -2392,7 +2392,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            index(indexName, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().getTotalHits(), equalTo(100L));
@@ -2555,9 +2555,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         createIndex("test-idx-1", "test-idx-2", "test-idx-3");
         logger.info("--> indexing some data");
         indexRandom(true,
-                client().prepareIndex("test-idx-1", "doc").setSource("foo", "bar"),
-                client().prepareIndex("test-idx-2", "doc").setSource("foo", "bar"),
-                client().prepareIndex("test-idx-3", "doc").setSource("foo", "bar"));
+                client().prepareIndex("test-idx-1", "_doc").setSource("foo", "bar"),
+                client().prepareIndex("test-idx-2", "_doc").setSource("foo", "bar"),
+                client().prepareIndex("test-idx-3", "_doc").setSource("foo", "bar"));
 
         logger.info("--> creating 2 snapshots");
         CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get();
@@ -2608,7 +2608,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         createIndex(indexName);
         ensureGreen();
         for (int i = 0; i < 10; i++) {
-            index(indexName, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
 
@@ -2625,7 +2625,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> index more documents");
         for (int i = 10; i < 20; i++) {
-            index(indexName, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
 
@@ -2690,7 +2690,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertAcked(prepareCreate(indexName, 1, Settings.builder().put("number_of_replicas", 0)));
         ensureGreen();
         for (int i = 0; i < 10; i++) {
-            index(indexName, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
 
@@ -2731,7 +2731,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         logger.info("--> take another snapshot to be in-progress");
         // add documents so there are data files to block on
         for (int i = 10; i < 20; i++) {
-            index(indexName, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
 
@@ -2821,7 +2821,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> indexing some data");
         for (int i = 0; i < numDocs; i++) {
-            index(index, "doc", Integer.toString(i), "foo", "bar" + i);
+            index(index, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
 
@@ -2884,7 +2884,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         ensureGreen();
         final int numDocs = randomIntBetween(1, 5);
         for (int i = 0; i < numDocs; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo((long) numDocs));
@@ -2940,7 +2940,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         ensureGreen();
         final int numDocs = randomIntBetween(1, 5);
         for (int i = 0; i < numDocs; i++) {
-            index("test-idx-good", "doc", Integer.toString(i), "foo", "bar" + i);
+            index("test-idx-good", "_doc", Integer.toString(i), "foo", "bar" + i);
         }
         refresh();
 
@@ -3019,7 +3019,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
                 if (randomBoolean()) {
                     final int numDocs = randomIntBetween(1, 5);
                     for (int k = 0; k < numDocs; k++) {
-                        index("test-idx-" + j, "doc", Integer.toString(k), "foo", "bar" + k);
+                        index("test-idx-" + j, "_doc", Integer.toString(k), "foo", "bar" + k);
                     }
                     refresh();
                 }
diff --git a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
index 6db737793fe..07c2dac0e7c 100644
--- a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
+++ b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
@@ -44,7 +44,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 
 public class SharedSignificantTermsTestMethods {
     public static final String INDEX_NAME = "testidx";
-    public static final String DOC_TYPE = "doc";
+    public static final String DOC_TYPE = "_doc";
     public static final String TEXT_FIELD = "text";
     public static final String CLASS_FIELD = "class";
 
@@ -82,7 +82,7 @@ public class SharedSignificantTermsTestMethods {
             textMappings += ",fielddata=true";
         }
         assertAcked(testCase.prepareCreate(INDEX_NAME).setSettings(settings, XContentType.JSON)
-                .addMapping("doc", "text", textMappings, CLASS_FIELD, "type=keyword"));
+                .addMapping("_doc", "text", textMappings, CLASS_FIELD, "type=keyword"));
         String[] gb = {"0", "1"};
         List indexRequestBuilderList = new ArrayList<>();
         indexRequestBuilderList.add(client().prepareIndex(INDEX_NAME, DOC_TYPE, "1")
diff --git a/core/src/test/resources/org/elasticsearch/search/query/all-query-index.json b/core/src/test/resources/org/elasticsearch/search/query/all-query-index.json
index 3b068132d51..72c9b54f6e3 100644
--- a/core/src/test/resources/org/elasticsearch/search/query/all-query-index.json
+++ b/core/src/test/resources/org/elasticsearch/search/query/all-query-index.json
@@ -6,7 +6,7 @@
     }
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "f1": {"type": "text"},
         "f2": {"type": "keyword"},
diff --git a/docs/build.gradle b/docs/build.gradle
index 65d717958ea..bca2a051000 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -70,7 +70,7 @@ Closure setupTwitter = { String name, int count ->
               number_of_shards: 1
               number_of_replicas: 1
             mappings:
-              tweet:
+              _doc:
                 properties:
                   user:
                     type: keyword
@@ -82,7 +82,7 @@ Closure setupTwitter = { String name, int count ->
   - do:
         bulk:
           index: twitter
-          type: tweet
+          type: _doc
           refresh: true
           body: |'''
     for (int i = 0; i < count; i++) {
@@ -134,7 +134,7 @@ buildRestTests.setups['ledger'] = '''
               number_of_shards: 2
               number_of_replicas: 1
             mappings:
-              sale:
+              _doc:
                 properties:
                   type:
                     type: keyword
@@ -143,7 +143,7 @@ buildRestTests.setups['ledger'] = '''
   - do:
         bulk:
           index: ledger
-          type: sale
+          type: _doc
           refresh: true
           body: |
             {"index":{}}
@@ -167,14 +167,14 @@ buildRestTests.setups['sales'] = '''
               number_of_shards: 2
               number_of_replicas: 1
             mappings:
-              sale:
+              _doc:
                 properties:
                   type:
                     type: keyword
   - do:
         bulk:
           index: sales
-          type: sale
+          type: _doc
           refresh: true
           body: |
             {"index":{}}
@@ -204,7 +204,7 @@ buildRestTests.setups['bank'] = '''
   - do:
         bulk:
           index: bank
-          type: account
+          type: _doc
           refresh: true
           body: |
 #bank_data#
@@ -231,7 +231,7 @@ buildRestTests.setups['range_index'] = '''
               number_of_shards: 2
               number_of_replicas: 1
             mappings:
-              my_type:
+              _doc:
                 properties:
                   expected_attendees:
                     type: integer_range
@@ -241,7 +241,7 @@ buildRestTests.setups['range_index'] = '''
   - do:
         bulk:
           index: range_index
-          type: my_type
+          type: _doc
           refresh: true
           body: |
             {"index":{"_id": 1}}
@@ -271,7 +271,7 @@ buildRestTests.setups['stackoverflow'] = '''
               number_of_shards: 1
               number_of_replicas: 1
             mappings:
-              question:
+              _doc:
                 properties:
                   author:
                     type: keyword
@@ -280,7 +280,7 @@ buildRestTests.setups['stackoverflow'] = '''
   - do:
         bulk:
           index: stackoverflow
-          type: question
+          type: _doc
           refresh: true
           body: |'''
 
@@ -326,7 +326,7 @@ buildRestTests.setups['news'] = '''
               number_of_shards: 1
               number_of_replicas: 1
             mappings:
-              question:
+              _doc:
                 properties:
                   source:
                     type: keyword
@@ -335,7 +335,7 @@ buildRestTests.setups['news'] = '''
   - do:
         bulk:
           index: news
-          type: article
+          type: _doc
           refresh: true
           body: |'''
 
@@ -379,14 +379,14 @@ buildRestTests.setups['exams'] = '''
               number_of_shards: 1
               number_of_replicas: 1
             mappings:
-              exam:
+              _doc:
                 properties:
                   grade:
                     type: byte
   - do:
         bulk:
           index: exams
-          type: exam
+          type: _doc
           refresh: true
           body: |
             {"index":{}}
@@ -444,7 +444,7 @@ buildRestTests.setups['analyze_sample'] = '''
                     type: custom
                     filter: [lowercase]
             mappings:
-              tweet:
+              _doc:
                 properties:
                   obj1.field1:
                     type: text'''
@@ -459,14 +459,14 @@ buildRestTests.setups['latency'] = '''
               number_of_shards: 1
               number_of_replicas: 1
             mappings:
-              data:
+              _doc:
                 properties:
                   load_time:
                     type: long
   - do:
         bulk:
           index: latency
-          type: data
+          type: _doc
           refresh: true
           body: |'''
 
@@ -491,14 +491,14 @@ buildRestTests.setups['iprange'] = '''
               number_of_shards: 1
               number_of_replicas: 1
             mappings:
-              data:
+              _doc:
                 properties:
                   ip:
                     type: ip
   - do:
         bulk:
           index: ip_addresses
-          type: data
+          type: _doc
           refresh: true
           body: |'''
 
diff --git a/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc b/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc
index 4029b3a2902..3e8117f41d4 100644
--- a/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc
@@ -33,7 +33,7 @@ Example:
 
 [source,js]
 --------------------------------------------------
-PUT /emails/message/_bulk?refresh
+PUT /emails/_doc/_bulk?refresh
 { "index" : { "_id" : 1 } }
 { "accounts" : ["hillary", "sidney"]}
 { "index" : { "_id" : 2 } }
@@ -41,7 +41,7 @@ PUT /emails/message/_bulk?refresh
 { "index" : { "_id" : 3 } }
 { "accounts" : ["vladimir", "donald"]}
 
-GET emails/message/_search
+GET emails/_search
 {
   "size": 0,
   "aggs" : {
diff --git a/docs/reference/aggregations/bucket/children-aggregation.asciidoc b/docs/reference/aggregations/bucket/children-aggregation.asciidoc
index 0503558ed23..e616359e8a8 100644
--- a/docs/reference/aggregations/bucket/children-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/children-aggregation.asciidoc
@@ -14,7 +14,7 @@ For example, let's say we have an index of questions and answers. The answer typ
 PUT child_example
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "join": {
           "type": "join",
@@ -37,7 +37,7 @@ An example of a question document:
 
 [source,js]
 --------------------------------------------------
-PUT child_example/doc/1
+PUT child_example/_doc/1
 {
   "join": {
     "name": "question"
@@ -58,7 +58,7 @@ Examples of `answer` documents:
 
 [source,js]
 --------------------------------------------------
-PUT child_example/doc/2?routing=1
+PUT child_example/_doc/2?routing=1
 {
   "join": {
     "name": "answer",
@@ -73,7 +73,7 @@ PUT child_example/doc/2?routing=1
   "creation_date": "2009-05-04T13:45:37.030"
 }
 
-PUT child_example/doc/3?routing=1&refresh
+PUT child_example/_doc/3?routing=1&refresh
 {
   "join": {
     "name": "answer",
diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
index 2cac8729b86..30ea2832a70 100644
--- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
@@ -125,12 +125,12 @@ Consider the following example:
 
 [source,js]
 ---------------------------------
-PUT my_index/log/1?refresh
+PUT my_index/_doc/1?refresh
 {
   "date": "2015-10-01T00:30:00Z"
 }
 
-PUT my_index/log/2?refresh
+PUT my_index/_doc/2?refresh
 {
   "date": "2015-10-01T01:30:00Z"
 }
@@ -247,12 +247,12 @@ to run from 6am to 6am:
 
 [source,js]
 -----------------------------
-PUT my_index/log/1?refresh
+PUT my_index/_doc/1?refresh
 {
   "date": "2015-10-01T05:30:00Z"
 }
 
-PUT my_index/log/2?refresh
+PUT my_index/_doc/2?refresh
 {
   "date": "2015-10-01T06:30:00Z"
 }
diff --git a/docs/reference/aggregations/bucket/filters-aggregation.asciidoc b/docs/reference/aggregations/bucket/filters-aggregation.asciidoc
index 97b7f2a50e1..3ca86d1d7a0 100644
--- a/docs/reference/aggregations/bucket/filters-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/filters-aggregation.asciidoc
@@ -9,7 +9,7 @@ Example:
 
 [source,js]
 --------------------------------------------------
-PUT /logs/message/_bulk?refresh
+PUT /logs/_doc/_bulk?refresh
 { "index" : { "_id" : 1 } }
 { "body" : "warning: page could not be rendered" }
 { "index" : { "_id" : 2 } }
@@ -134,7 +134,7 @@ The following snippet shows a response where the `other` bucket is requested to
 
 [source,js]
 --------------------------------------------------
-PUT logs/message/4?refresh
+PUT logs/_doc/4?refresh
 {
   "body": "info: user Bob logged out"
 }
diff --git a/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc b/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc
index 2f099cc8b2f..c8bd896b037 100644
--- a/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc
@@ -7,7 +7,7 @@ Example:
 
 [source,js]
 --------------------------------------------------
-GET /ip_addresses/data/_search
+GET /ip_addresses/_search
 {
     "size": 10,
     "aggs" : {
@@ -55,7 +55,7 @@ IP ranges can also be defined as CIDR masks:
 
 [source,js]
 --------------------------------------------------
-GET /ip_addresses/data/_search
+GET /ip_addresses/_search
 {
     "size": 0,
     "aggs" : {
@@ -109,7 +109,7 @@ Setting the `keyed` flag to `true` will associate a unique string key with each
 
 [source,js]
 --------------------------------------------------
-GET /ip_addresses/data/_search
+GET /ip_addresses/_search
 {
     "size": 0,
     "aggs": {
@@ -158,7 +158,7 @@ It is also possible to customize the key for each range:
 
 [source,js]
 --------------------------------------------------
-GET /ip_addresses/data/_search
+GET /ip_addresses/_search
 {
     "size": 0,
     "aggs": {
@@ -201,4 +201,4 @@ Response:
     }
 }
 --------------------------------------------------
-// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
\ No newline at end of file
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
diff --git a/docs/reference/aggregations/matrix/stats-aggregation.asciidoc b/docs/reference/aggregations/matrix/stats-aggregation.asciidoc
index 3cc207fef7d..df5afc82523 100644
--- a/docs/reference/aggregations/matrix/stats-aggregation.asciidoc
+++ b/docs/reference/aggregations/matrix/stats-aggregation.asciidoc
@@ -17,13 +17,13 @@ The `matrix_stats` aggregation is a numeric aggregation that computes the follow
 
 [source,js]
 --------------------------------------------------
-PUT /statistics/doc/0
+PUT /statistics/_doc/0
 {"poverty": 24.0, "income": 50000.0}
 
-PUT /statistics/doc/1
+PUT /statistics/_doc/1
 {"poverty": 13.0, "income": 95687.0}
 
-PUT /statistics/doc/2
+PUT /statistics/_doc/2
 {"poverty": 69.0, "income": 7890.0}
 
 POST /_refresh
diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
index 3e06c6f347f..938d42a70ab 100644
--- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
@@ -50,7 +50,7 @@ POST /sales/_search?size=0
     "aggs" : {
         "type_count" : {
             "cardinality" : {
-                "field" : "type",
+                "field" : "_doc",
                 "precision_threshold": 100 <1>
             }
         }
@@ -207,7 +207,7 @@ POST /sales/_search?size=0
                 "script" : {
                     "id": "my_script",
                     "params": {
-                        "type_field": "type",
+                        "type_field": "_doc",
                         "promoted_field": "promoted"
                     }
                 }
diff --git a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc
index 9db4b7b7c35..0265968d58a 100644
--- a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc
@@ -26,7 +26,7 @@ Let's look at a range of percentiles representing load time:
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {
@@ -78,7 +78,7 @@ must be a value between 0-100 inclusive):
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {
@@ -101,7 +101,7 @@ By default the `keyed` flag is set to `true` which associates a unique string ke
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs": {
@@ -170,7 +170,7 @@ a script to convert them on-the-fly:
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {
@@ -199,7 +199,7 @@ This will interpret the `script` parameter as an `inline` script with the `painl
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {
@@ -264,7 +264,7 @@ This balance can be controlled using a `compression` parameter:
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {
@@ -315,7 +315,7 @@ The HDR Histogram can be used by specifying the `method` parameter in the reques
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {
@@ -348,7 +348,7 @@ had a value.
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {
diff --git a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc
index 58266ffa7db..da5595adfbc 100644
--- a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc
@@ -24,7 +24,7 @@ Let's look at a range of percentiles representing load time:
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {
@@ -69,7 +69,7 @@ By default the `keyed` flag is set to `true` associates a unique string key with
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs": {
@@ -120,7 +120,7 @@ a script to convert them on-the-fly:
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {
@@ -149,7 +149,7 @@ This will interpret the `script` parameter as an `inline` script with the `painl
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {
@@ -185,7 +185,7 @@ The HDR Histogram can be used by specifying the `method` parameter in the reques
 
 [source,js]
 --------------------------------------------------
-GET latency/data/_search
+GET latency/_search
 {
     "size": 0,
     "aggs" : {
diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
index dff8bd68bbd..daa86969e45 100644
--- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
@@ -151,7 +151,7 @@ Imagine a situation where you index the following documents into an index with 2
 
 [source,js]
 --------------------------------------------------
-PUT /transactions/stock/_bulk?refresh
+PUT /transactions/_doc/_bulk?refresh
 {"index":{"_id":1}}
 {"type": "sale","amount": 80}
 {"index":{"_id":2}}
diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
index b2df9bacae2..e5966e56b35 100644
--- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
@@ -86,7 +86,7 @@ Possible response:
                    "hits": [
                       {
                          "_index": "sales",
-                         "_type": "sale",
+                         "_type": "_doc",
                          "_id": "AVnNBmauCQpcRyxw6ChK",
                          "_source": {
                             "date": "2015/03/01 00:00:00",
@@ -111,7 +111,7 @@ Possible response:
                    "hits": [
                       {
                          "_index": "sales",
-                         "_type": "sale",
+                         "_type": "_doc",
                          "_id": "AVnNBmauCQpcRyxw6ChL",
                          "_source": {
                             "date": "2015/03/01 00:00:00",
@@ -136,7 +136,7 @@ Possible response:
                    "hits": [
                       {
                          "_index": "sales",
-                         "_type": "sale",
+                         "_type": "_doc",
                          "_id": "AVnNBmatCQpcRyxw6ChH",
                          "_source": {
                             "date": "2015/01/01 00:00:00",
@@ -234,7 +234,7 @@ Let's see how it works with a real sample. Considering the following mapping:
 PUT /sales
 {
     "mappings": {
-        "product" : {
+        "_doc" : {
             "properties" : {
                 "tags" : { "type" : "keyword" },
                 "comments" : { <1>
@@ -256,7 +256,7 @@ And some documents:
 
 [source,js]
 --------------------------------------------------
-PUT /sales/product/1?refresh
+PUT /sales/_doc/1?refresh
 {
     "tags": ["car", "auto"],
     "comments": [
@@ -324,7 +324,7 @@ Top hits response snippet with a nested hit, which resides in the first slot of
                 "hits": [
                   {
                     "_index": "sales",
-                    "_type" : "product",
+                    "_type" : "_doc",
                     "_id": "1",
                     "_nested": {
                       "field": "comments",  <1>
@@ -392,4 +392,4 @@ the second slow of the `nested_child_field` field:
 }
 ...
 --------------------------------------------------
-// NOTCONSOLE
\ No newline at end of file
+// NOTCONSOLE
diff --git a/docs/reference/aggregations/misc.asciidoc b/docs/reference/aggregations/misc.asciidoc
index 5998c0c12cf..20d234302a6 100644
--- a/docs/reference/aggregations/misc.asciidoc
+++ b/docs/reference/aggregations/misc.asciidoc
@@ -17,7 +17,7 @@ setting `size=0`. For example:
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/_search
+GET /twitter/_search
 {
   "size": 0,
   "aggregations": {
@@ -44,7 +44,7 @@ Consider this example where we want to associate the color blue with our `terms`
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/_search
+GET /twitter/_search
 {
   "size": 0,
   "aggs": {
@@ -96,7 +96,7 @@ Considering the following <> field in a mapping can specify its own
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "title": {
           "type":     "text",
diff --git a/docs/reference/analysis/analyzers/configuring.asciidoc b/docs/reference/analysis/analyzers/configuring.asciidoc
index d4c606df24d..8ed4c09b4d3 100644
--- a/docs/reference/analysis/analyzers/configuring.asciidoc
+++ b/docs/reference/analysis/analyzers/configuring.asciidoc
@@ -21,7 +21,7 @@ PUT my_index
     }
   },
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "my_text": {
           "type":     "text",
diff --git a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
index 9386d7d9d35..6e881121a0f 100644
--- a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
+++ b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
@@ -125,7 +125,7 @@ PUT my_index
     }
   },
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "text": {
           "type": "text",
@@ -205,7 +205,7 @@ the length of the original text:
 
 [source,js]
 ----------------------------
-PUT my_index/my_type/1?refresh
+PUT my_index/_doc/1?refresh
 {
   "text": "The fooBarBaz method"
 }
@@ -246,7 +246,7 @@ The output from the above is:
     "hits": [
       {
         "_index": "my_index",
-        "_type": "my_type",
+        "_type": "_doc",
         "_id": "1",
         "_score": 0.2876821,
         "_source": {
diff --git a/docs/reference/analysis/testing.asciidoc b/docs/reference/analysis/testing.asciidoc
index 0a603973e18..c8c7c3fa2b4 100644
--- a/docs/reference/analysis/testing.asciidoc
+++ b/docs/reference/analysis/testing.asciidoc
@@ -58,7 +58,7 @@ PUT my_index
     }
   },
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "my_text": {
           "type": "text",
diff --git a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
index 5c2359163f9..3cf1f8403e2 100644
--- a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
@@ -250,7 +250,7 @@ PUT my_index
     }
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "title": {
           "type": "text",
@@ -262,7 +262,7 @@ PUT my_index
   }
 }
 
-PUT my_index/doc/1
+PUT my_index/_doc/1
 {
   "title": "Quick Foxes" <1>
 }
@@ -305,7 +305,7 @@ GET my_index/_search
     "hits": [
       {
         "_index": "my_index",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 0.5753642,
         "_source": {
diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc
index 6c90df9de40..adf48bb0c76 100644
--- a/docs/reference/docs/bulk.asciidoc
+++ b/docs/reference/docs/bulk.asciidoc
@@ -57,10 +57,10 @@ newlines. Example:
 [source,js]
 --------------------------------------------------
 $ cat requests
-{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
+{ "index" : { "_index" : "test", "_type" : "_doc", "_id" : "1" } }
 { "field1" : "value1" }
 $ curl -s -H "Content-Type: application/x-ndjson" -XPOST localhost:9200/_bulk --data-binary "@requests"; echo
-{"took":7, "errors": false, "items":[{"index":{"_index":"test","_type":"type1","_id":"1","_version":1,"result":"created","forced_refresh":false}}]}
+{"took":7, "errors": false, "items":[{"index":{"_index":"test","_type":"_doc","_id":"1","_version":1,"result":"created","forced_refresh":false}}]}
 --------------------------------------------------
 // NOTCONSOLE
 // Not converting to console because this shows how curl works
@@ -72,12 +72,12 @@ example of a correct sequence of bulk commands:
 [source,js]
 --------------------------------------------------
 POST _bulk
-{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
+{ "index" : { "_index" : "test", "_type" : "_doc", "_id" : "1" } }
 { "field1" : "value1" }
-{ "delete" : { "_index" : "test", "_type" : "type1", "_id" : "2" } }
-{ "create" : { "_index" : "test", "_type" : "type1", "_id" : "3" } }
+{ "delete" : { "_index" : "test", "_type" : "_doc", "_id" : "2" } }
+{ "create" : { "_index" : "test", "_type" : "_doc", "_id" : "3" } }
 { "field1" : "value3" }
-{ "update" : {"_id" : "1", "_type" : "type1", "_index" : "test"} }
+{ "update" : {"_id" : "1", "_type" : "_doc", "_index" : "test"} }
 { "doc" : {"field2" : "value2"} }
 --------------------------------------------------
 // CONSOLE
@@ -93,7 +93,7 @@ The result of this bulk operation is:
       {
          "index": {
             "_index": "test",
-            "_type": "type1",
+            "_type": "_doc",
             "_id": "1",
             "_version": 1,
             "result": "created",
@@ -110,7 +110,7 @@ The result of this bulk operation is:
       {
          "delete": {
             "_index": "test",
-            "_type": "type1",
+            "_type": "_doc",
             "_id": "2",
             "_version": 1,
             "result": "not_found",
@@ -127,7 +127,7 @@ The result of this bulk operation is:
       {
          "create": {
             "_index": "test",
-            "_type": "type1",
+            "_type": "_doc",
             "_id": "3",
             "_version": 1,
             "result": "created",
@@ -144,7 +144,7 @@ The result of this bulk operation is:
       {
          "update": {
             "_index": "test",
-            "_type": "type1",
+            "_type": "_doc",
             "_id": "1",
             "_version": 2,
             "result": "updated",
@@ -246,15 +246,15 @@ the options. Example with update actions:
 [source,js]
 --------------------------------------------------
 POST _bulk
-{ "update" : {"_id" : "1", "_type" : "type1", "_index" : "index1", "retry_on_conflict" : 3} }
+{ "update" : {"_id" : "1", "_type" : "_doc", "_index" : "index1", "retry_on_conflict" : 3} }
 { "doc" : {"field" : "value"} }
-{ "update" : { "_id" : "0", "_type" : "type1", "_index" : "index1", "retry_on_conflict" : 3} }
+{ "update" : { "_id" : "0", "_type" : "_doc", "_index" : "index1", "retry_on_conflict" : 3} }
 { "script" : { "source": "ctx._source.counter += params.param1", "lang" : "painless", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}}
-{ "update" : {"_id" : "2", "_type" : "type1", "_index" : "index1", "retry_on_conflict" : 3} }
+{ "update" : {"_id" : "2", "_type" : "_doc", "_index" : "index1", "retry_on_conflict" : 3} }
 { "doc" : {"field" : "value"}, "doc_as_upsert" : true }
-{ "update" : {"_id" : "3", "_type" : "type1", "_index" : "index1", "_source" : true} }
+{ "update" : {"_id" : "3", "_type" : "_doc", "_index" : "index1", "_source" : true} }
 { "doc" : {"field" : "value"} }
-{ "update" : {"_id" : "4", "_type" : "type1", "_index" : "index1"} }
+{ "update" : {"_id" : "4", "_type" : "_doc", "_index" : "index1"} }
 { "doc" : {"field" : "value"}, "_source": true}
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc
index 0aea249d899..cbcbc29e165 100644
--- a/docs/reference/docs/delete-by-query.asciidoc
+++ b/docs/reference/docs/delete-by-query.asciidoc
@@ -76,7 +76,7 @@ will only delete `tweet` documents from the `twitter` index:
 
 [source,js]
 --------------------------------------------------
-POST twitter/tweet/_delete_by_query?conflicts=proceed
+POST twitter/_doc/_delete_by_query?conflicts=proceed
 {
   "query": {
     "match_all": {}
diff --git a/docs/reference/docs/delete.asciidoc b/docs/reference/docs/delete.asciidoc
index 2eb2b88ed13..b8cf89a2410 100644
--- a/docs/reference/docs/delete.asciidoc
+++ b/docs/reference/docs/delete.asciidoc
@@ -8,7 +8,7 @@ from an index called twitter, under a type called tweet, with id valued
 
 [source,js]
 --------------------------------------------------
-DELETE /twitter/tweet/1
+DELETE /twitter/_doc/1
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -24,7 +24,7 @@ The result of the above delete operation is:
         "successful" : 2
     },
     "_index" : "twitter",
-    "_type" : "tweet",
+    "_type" : "_doc",
     "_id" : "1",
     "_version" : 2,
     "_primary_term": 1,
@@ -59,7 +59,7 @@ Example to delete with routing
 
 [source,js]
 --------------------------------------------------
-PUT /twitter/tweet/1?routing=kimchy
+PUT /twitter/_doc/1?routing=kimchy
 {
     "test": "test"
 }
@@ -70,7 +70,7 @@ PUT /twitter/tweet/1?routing=kimchy
 
 [source,js]
 --------------------------------------------------
-DELETE /twitter/tweet/1?routing=kimchy
+DELETE /twitter/_doc/1?routing=kimchy
 --------------------------------------------------
 // CONSOLE
 // TEST[continued]
@@ -136,7 +136,7 @@ to 5 minutes:
 
 [source,js]
 --------------------------------------------------
-DELETE /twitter/tweet/1?timeout=5m
+DELETE /twitter/_doc/1?timeout=5m
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
diff --git a/docs/reference/docs/get.asciidoc b/docs/reference/docs/get.asciidoc
index 11b2347e7f3..81c4bc306b2 100644
--- a/docs/reference/docs/get.asciidoc
+++ b/docs/reference/docs/get.asciidoc
@@ -7,7 +7,7 @@ twitter, under a type called tweet, with id valued 0:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/0
+GET twitter/_doc/0
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -18,7 +18,7 @@ The result of the above get operation is:
 --------------------------------------------------
 {
     "_index" : "twitter",
-    "_type" : "tweet",
+    "_type" : "_doc",
     "_id" : "0",
     "_version" : 1,
     "found": true,
@@ -42,7 +42,7 @@ The API also allows to check for the existence of a document using
 
 [source,js]
 --------------------------------------------------
-HEAD twitter/tweet/0
+HEAD twitter/_doc/0
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -68,7 +68,7 @@ You can turn off `_source` retrieval by using the `_source` parameter:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/0?_source=false
+GET twitter/_doc/0?_source=false
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -80,7 +80,7 @@ of fields or wildcard expressions. Example:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/0?_source_include=*.id&_source_exclude=entities
+GET twitter/_doc/0?_source_include=*.id&_source_exclude=entities
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -89,7 +89,7 @@ If you only want to specify includes, you can use a shorter notation:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/0?_source=*.id,retweeted
+GET twitter/_doc/0?_source=*.id,retweeted
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -108,7 +108,7 @@ Consider for instance the following mapping:
 PUT twitter
 {
    "mappings": {
-      "tweet": {
+      "_doc": {
          "properties": {
             "counter": {
                "type": "integer",
@@ -129,7 +129,7 @@ Now we can add a document:
 
 [source,js]
 --------------------------------------------------
-PUT twitter/tweet/1
+PUT twitter/_doc/1
 {
     "counter" : 1,
     "tags" : ["red"]
@@ -142,7 +142,7 @@ PUT twitter/tweet/1
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/1?stored_fields=tags,counter
+GET twitter/_doc/1?stored_fields=tags,counter
 --------------------------------------------------
 // CONSOLE
 // TEST[continued]
@@ -153,7 +153,7 @@ The result of the above get operation is:
 --------------------------------------------------
 {
    "_index": "twitter",
-   "_type": "tweet",
+   "_type": "_doc",
    "_id": "1",
    "_version": 1,
    "found": true,
@@ -174,7 +174,7 @@ It is also possible to retrieve metadata fields like the `_routing` field:
 
 [source,js]
 --------------------------------------------------
-PUT twitter/tweet/2?routing=user1
+PUT twitter/_doc/2?routing=user1
 {
     "counter" : 1,
     "tags" : ["white"]
@@ -185,7 +185,7 @@ PUT twitter/tweet/2?routing=user1
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/2?routing=user1&stored_fields=tags,counter
+GET twitter/_doc/2?routing=user1&stored_fields=tags,counter
 --------------------------------------------------
 // CONSOLE
 // TEST[continued]
@@ -196,7 +196,7 @@ The result of the above get operation is:
 --------------------------------------------------
 {
    "_index": "twitter",
-   "_type": "tweet",
+   "_type": "_doc",
    "_id": "2",
    "_version": 1,
    "_routing": "user1",
@@ -223,7 +223,7 @@ without any additional content around it. For example:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/1/_source
+GET twitter/_doc/1/_source
 --------------------------------------------------
 // CONSOLE
 // TEST[continued]
@@ -232,7 +232,7 @@ You can also use the same source filtering parameters to control which parts of
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/1/_source?_source_include=*.id&_source_exclude=entities'
+GET twitter/_doc/1/_source?_source_include=*.id&_source_exclude=entities'
 --------------------------------------------------
 // CONSOLE
 // TEST[continued]
@@ -242,7 +242,7 @@ An existing document will not have a _source if it is disabled in the <>.
 
 The index API adds or updates a typed JSON document in a specific index,
 making it searchable. The following example inserts the JSON document
-into the "twitter" index, under a type called "tweet" with an id of 1:
+into the "twitter" index, under a type called "_doc" with an id of 1:
 
 [source,js]
 --------------------------------------------------
-PUT twitter/tweet/1
+PUT twitter/_doc/1
 {
     "user" : "kimchy",
     "post_date" : "2009-11-15T14:12:12",
@@ -29,7 +29,7 @@ The result of the above index operation is:
         "successful" : 2
     },
     "_index" : "twitter",
-    "_type" : "tweet",
+    "_type" : "_doc",
     "_id" : "1",
     "_version" : 1,
     "_seq_no" : 0,
@@ -96,7 +96,7 @@ meantime. For example:
 
 [source,js]
 --------------------------------------------------
-PUT twitter/tweet/1?version=2
+PUT twitter/_doc/1?version=2
 {
     "message" : "elasticsearch now has versioning support, double cool!"
 }
@@ -176,7 +176,7 @@ Here is an example of using the `op_type` parameter:
 
 [source,js]
 --------------------------------------------------
-PUT twitter/tweet/1?op_type=create
+PUT twitter/_doc/1?op_type=create
 {
     "user" : "kimchy",
     "post_date" : "2009-11-15T14:12:12",
@@ -189,7 +189,7 @@ Another option to specify `create` is to use the following uri:
 
 [source,js]
 --------------------------------------------------
-PUT twitter/tweet/1/_create
+PUT twitter/_doc/1/_create
 {
     "user" : "kimchy",
     "post_date" : "2009-11-15T14:12:12",
@@ -208,7 +208,7 @@ will automatically be set to `create`. Here is an example (note the
 
 [source,js]
 --------------------------------------------------
-POST twitter/tweet/
+POST twitter/_doc/
 {
     "user" : "kimchy",
     "post_date" : "2009-11-15T14:12:12",
@@ -228,7 +228,7 @@ The result of the above index operation is:
         "successful" : 2
     },
     "_index" : "twitter",
-    "_type" : "tweet",
+    "_type" : "_doc",
     "_id" : "6a8ca01c-7896-48e9-81cc-9f70661fcb32",
     "_version" : 1,
     "_seq_no" : 0,
@@ -258,7 +258,7 @@ POST twitter/tweet?routing=kimchy
 --------------------------------------------------
 // CONSOLE
 
-In the example above, the "tweet" document is routed to a shard based on
+In the example above, the "_doc" document is routed to a shard based on
 the `routing` parameter provided: "kimchy".
 
 When setting up explicit mapping, the `_routing` field can be optionally
@@ -372,7 +372,7 @@ to 5 minutes:
 
 [source,js]
 --------------------------------------------------
-PUT twitter/tweet/1?timeout=5m
+PUT twitter/_doc/1?timeout=5m
 {
     "user" : "kimchy",
     "post_date" : "2009-11-15T14:12:12",
diff --git a/docs/reference/docs/multi-get.asciidoc b/docs/reference/docs/multi-get.asciidoc
index b241607f853..639090a1c1b 100644
--- a/docs/reference/docs/multi-get.asciidoc
+++ b/docs/reference/docs/multi-get.asciidoc
@@ -14,12 +14,12 @@ GET /_mget
     "docs" : [
         {
             "_index" : "test",
-            "_type" : "type",
+            "_type" : "_doc",
             "_id" : "1"
         },
         {
             "_index" : "test",
-            "_type" : "type",
+            "_type" : "_doc",
             "_id" : "2"
         }
     ]
@@ -36,11 +36,11 @@ GET /test/_mget
 {
     "docs" : [
         {
-            "_type" : "type",
+            "_type" : "_doc",
             "_id" : "1"
         },
         {
-            "_type" : "type",
+            "_type" : "_doc",
             "_id" : "2"
         }
     ]
@@ -78,48 +78,6 @@ GET /test/type/_mget
 --------------------------------------------------
 // CONSOLE
 
-[float]
-[[mget-type]]
-=== Optional Type
-
-The mget API allows for `_type` to be optional. Set it to `_all` or leave it empty in order
-to fetch the first document matching the id across all types.
-
-If you don't set the type and have many documents sharing the same `_id`, you will end up
-getting only the first matching document.
-
-For example, if you have a document 1 within typeA and typeB then following request
-will give you back only the same document twice:
-
-[source,js]
---------------------------------------------------
-GET /test/_mget
-{
-    "ids" : ["1", "1"]
-}
---------------------------------------------------
-// CONSOLE
-
-You need in that case to explicitly set the `_type`:
-
-[source,js]
---------------------------------------------------
-GET /test/_mget/
-{
-  "docs" : [
-        {
-            "_type":"typeA",
-            "_id" : "1"
-        },
-        {
-            "_type":"typeB",
-            "_id" : "1"
-        }
-    ]
-}
---------------------------------------------------
-// CONSOLE
-
 [float]
 [[mget-source-filtering]]
 === Source filtering
@@ -139,19 +97,19 @@ GET /_mget
     "docs" : [
         {
             "_index" : "test",
-            "_type" : "type",
+            "_type" : "_doc",
             "_id" : "1",
             "_source" : false
         },
         {
             "_index" : "test",
-            "_type" : "type",
+            "_type" : "_doc",
             "_id" : "2",
             "_source" : ["field3", "field4"]
         },
         {
             "_index" : "test",
-            "_type" : "type",
+            "_type" : "_doc",
             "_id" : "3",
             "_source" : {
                 "include": ["user"],
@@ -178,13 +136,13 @@ GET /_mget
     "docs" : [
         {
             "_index" : "test",
-            "_type" : "type",
+            "_type" : "_doc",
             "_id" : "1",
             "stored_fields" : ["field1", "field2"]
         },
         {
             "_index" : "test",
-            "_type" : "type",
+            "_type" : "_doc",
             "_id" : "2",
             "stored_fields" : ["field3", "field4"]
         }
@@ -228,13 +186,13 @@ GET /_mget?routing=key1
     "docs" : [
         {
             "_index" : "test",
-            "_type" : "type",
+            "_type" : "_doc",
             "_id" : "1",
             "routing" : "key2"
         },
         {
             "_index" : "test",
-            "_type" : "type",
+            "_type" : "_doc",
             "_id" : "2"
         }
     ]
diff --git a/docs/reference/docs/multi-termvectors.asciidoc b/docs/reference/docs/multi-termvectors.asciidoc
index 19867478084..3e87d1d3c97 100644
--- a/docs/reference/docs/multi-termvectors.asciidoc
+++ b/docs/reference/docs/multi-termvectors.asciidoc
@@ -17,13 +17,13 @@ POST /_mtermvectors
    "docs": [
       {
          "_index": "twitter",
-         "_type": "tweet",
+         "_type": "_doc",
          "_id": "2",
          "term_statistics": true
       },
       {
          "_index": "twitter",
-         "_type": "tweet",
+         "_type": "_doc",
          "_id": "1",
          "fields": [
             "message"
@@ -46,7 +46,7 @@ POST /twitter/_mtermvectors
 {
    "docs": [
       {
-         "_type": "tweet",
+         "_type": "_doc",
          "_id": "2",
          "fields": [
             "message"
@@ -54,7 +54,7 @@ POST /twitter/_mtermvectors
          "term_statistics": true
       },
       {
-         "_type": "tweet",
+         "_type": "_doc",
          "_id": "1"
       }
    ]
@@ -67,7 +67,7 @@ And type:
 
 [source,js]
 --------------------------------------------------
-POST /twitter/tweet/_mtermvectors
+POST /twitter/_doc/_mtermvectors
 {
    "docs": [
       {
@@ -90,7 +90,7 @@ If all requested documents are on same index and have same type and also the par
 
 [source,js]
 --------------------------------------------------
-POST /twitter/tweet/_mtermvectors
+POST /twitter/_doc/_mtermvectors
 {
     "ids" : ["1", "2"],
     "parameters": {
@@ -115,7 +115,7 @@ POST /_mtermvectors
    "docs": [
       {
          "_index": "twitter",
-         "_type": "tweet",
+         "_type": "_doc",
          "doc" : {
             "user" : "John Doe",
             "message" : "twitter test test test"
@@ -123,7 +123,7 @@ POST /_mtermvectors
       },
       {
          "_index": "twitter",
-         "_type": "test",
+         "_type": "_doc",
          "doc" : {
            "user" : "Jane Doe",
            "message" : "Another twitter test ..."
diff --git a/docs/reference/docs/refresh.asciidoc b/docs/reference/docs/refresh.asciidoc
index 90c5d4e3afe..f301a8ec3da 100644
--- a/docs/reference/docs/refresh.asciidoc
+++ b/docs/reference/docs/refresh.asciidoc
@@ -84,9 +84,9 @@ These will create a document and immediately refresh the index so it is visible:
 
 [source,js]
 --------------------------------------------------
-PUT /test/test/1?refresh
+PUT /test/_doc/1?refresh
 {"test": "test"}
-PUT /test/test/2?refresh=true
+PUT /test/_doc/2?refresh=true
 {"test": "test"}
 --------------------------------------------------
 // CONSOLE
@@ -96,9 +96,9 @@ search:
 
 [source,js]
 --------------------------------------------------
-PUT /test/test/3
+PUT /test/_doc/3
 {"test": "test"}
-PUT /test/test/4?refresh=false
+PUT /test/_doc/4?refresh=false
 {"test": "test"}
 --------------------------------------------------
 // CONSOLE
@@ -107,7 +107,7 @@ This will create a document and wait for it to become visible for search:
 
 [source,js]
 --------------------------------------------------
-PUT /test/test/4?refresh=wait_for
+PUT /test/_doc/4?refresh=wait_for
 {"test": "test"}
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 77ff6162dcf..54631195fe1 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -144,7 +144,7 @@ POST _reindex
 {
   "source": {
     "index": "twitter",
-    "type": "tweet",
+    "type": "_doc",
     "query": {
       "term": {
         "user": "kimchy"
@@ -173,7 +173,7 @@ POST _reindex
 {
   "source": {
     "index": ["twitter", "blog"],
-    "type": ["tweet", "post"]
+    "type": ["_doc", "post"]
   },
   "dest": {
     "index": "all_together"
@@ -236,7 +236,7 @@ POST _reindex
 {
   "source": {
     "index": "twitter",
-    "_source": ["user", "tweet"]
+    "_source": ["user", "_doc"]
   },
   "dest": {
     "index": "new_twitter"
@@ -796,7 +796,7 @@ create an index containing documents that look like this:
 
 [source,js]
 --------------------------------------------------
-POST test/test/1?refresh
+POST test/_doc/1?refresh
 {
   "text": "words words",
   "flag": "foo"
@@ -829,7 +829,7 @@ Now you can get the new document:
 
 [source,js]
 --------------------------------------------------
-GET test2/test/1
+GET test2/_doc/1
 --------------------------------------------------
 // CONSOLE
 // TEST[continued]
@@ -842,7 +842,7 @@ and it'll look like:
   "found": true,
   "_id": "1",
   "_index": "test2",
-  "_type": "test",
+  "_type": "_doc",
   "_version": 1,
   "_source": {
     "text": "words words",
@@ -1023,9 +1023,9 @@ Assuming you have indices consisting of documents as following:
 
 [source,js]
 ----------------------------------------------------------------
-PUT metricbeat-2016.05.30/beat/1?refresh
+PUT metricbeat-2016.05.30/_doc/1?refresh
 {"system.cpu.idle.pct": 0.908}
-PUT metricbeat-2016.05.31/beat/1?refresh
+PUT metricbeat-2016.05.31/_doc/1?refresh
 {"system.cpu.idle.pct": 0.105}
 ----------------------------------------------------------------
 // CONSOLE
@@ -1061,8 +1061,8 @@ All documents from the previous metricbeat indices now can be found in the `*-1`
 
 [source,js]
 ----------------------------------------------------------------
-GET metricbeat-2016.05.30-1/beat/1
-GET metricbeat-2016.05.31-1/beat/1
+GET metricbeat-2016.05.30-1/_doc/1
+GET metricbeat-2016.05.31-1/_doc/1
 ----------------------------------------------------------------
 // CONSOLE
 // TEST[continued]
diff --git a/docs/reference/docs/termvectors.asciidoc b/docs/reference/docs/termvectors.asciidoc
index 05372d13f30..3cd21b21df4 100644
--- a/docs/reference/docs/termvectors.asciidoc
+++ b/docs/reference/docs/termvectors.asciidoc
@@ -8,7 +8,7 @@ realtime. This can be changed by setting `realtime` parameter to `false`.
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/1/_termvectors
+GET /twitter/_doc/1/_termvectors
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -18,7 +18,7 @@ retrieved either with a parameter in the url
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/1/_termvectors?fields=message
+GET /twitter/_doc/1/_termvectors?fields=message
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -130,7 +130,7 @@ First, we create an index that stores term vectors, payloads etc. :
 --------------------------------------------------
 PUT /twitter/
 { "mappings": {
-    "tweet": {
+    "_doc": {
       "properties": {
         "text": {
           "type": "text",
@@ -172,13 +172,13 @@ Second, we add some documents:
 
 [source,js]
 --------------------------------------------------
-PUT /twitter/tweet/1
+PUT /twitter/_doc/1
 {
   "fullname" : "John Doe",
   "text" : "twitter test test test "
 }
 
-PUT /twitter/tweet/2
+PUT /twitter/_doc/2
 {
   "fullname" : "Jane Doe",
   "text" : "Another twitter test ..."
@@ -192,7 +192,7 @@ The following request returns all information and statistics for field
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/1/_termvectors
+GET /twitter/_doc/1/_termvectors
 {
   "fields" : ["text"],
   "offsets" : true,
@@ -212,7 +212,7 @@ Response:
 {
     "_id": "1",
     "_index": "twitter",
-    "_type": "tweet",
+    "_type": "_doc",
     "_version": 1,
     "found": true,
     "took": 6,
@@ -280,7 +280,7 @@ Note that for the field `text`, the terms are not re-generated.
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/1/_termvectors
+GET /twitter/_doc/1/_termvectors
 {
   "fields" : ["text", "some_field_without_term_vectors"],
   "offsets" : true,
@@ -306,7 +306,7 @@ mapping will be dynamically created.*
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/_termvectors
+GET /twitter/_doc/_termvectors
 {
   "doc" : {
     "fullname" : "John Doe",
@@ -329,7 +329,7 @@ vectors, the term vectors will be re-generated.
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/_termvectors
+GET /twitter/_doc/_termvectors
 {
   "doc" : {
     "fullname" : "John Doe",
@@ -350,7 +350,7 @@ Response:
 --------------------------------------------------
 {
   "_index": "twitter",
-  "_type": "tweet",
+  "_type": "_doc",
   "_version": 0,
   "found": true,
   "took": 6,
@@ -396,7 +396,7 @@ their tf-idf must be too low.
 
 [source,js]
 --------------------------------------------------
-GET /imdb/movies/_termvectors
+GET /imdb/_doc/_termvectors
 {
     "doc": {
       "plot": "When wealthy industrialist Tony Stark is forced to build an armored suit after a life-threatening incident, he ultimately decides to use its technology to fight against evil."
@@ -421,7 +421,7 @@ Response:
 --------------------------------------------------
 {
    "_index": "imdb",
-   "_type": "movies",
+   "_type": "_doc",
    "_version": 0,
    "found": true,
    "term_vectors": {
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index 59e82ff36c4..55044dffed9 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -68,7 +68,7 @@ will only update `tweet` documents from the `twitter` index:
 
 [source,js]
 --------------------------------------------------
-POST twitter/tweet/_update_by_query?conflicts=proceed
+POST twitter/_doc/_update_by_query?conflicts=proceed
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -626,7 +626,7 @@ added a mapping value to pick up more fields from the data:
 PUT test
 {
   "mappings": {
-    "test": {
+    "_doc": {
       "dynamic": false,   <1>
       "properties": {
         "text": {"type": "text"}
@@ -635,17 +635,17 @@ PUT test
   }
 }
 
-POST test/test?refresh
+POST test/_doc?refresh
 {
   "text": "words words",
   "flag": "bar"
 }
-POST test/test?refresh
+POST test/_doc?refresh
 {
   "text": "words words",
   "flag": "foo"
 }
-PUT test/_mapping/test   <2>
+PUT test/_mapping/_doc   <2>
 {
   "properties": {
     "text": {"type": "text"},
diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc
index 9e3a537e96b..6a0f09c982b 100644
--- a/docs/reference/docs/update.asciidoc
+++ b/docs/reference/docs/update.asciidoc
@@ -17,7 +17,7 @@ For example, let's index a simple doc:
 
 [source,js]
 --------------------------------------------------
-PUT test/type1/1
+PUT test/_doc/1
 {
     "counter" : 1,
     "tags" : ["red"]
@@ -32,7 +32,7 @@ Now, we can execute a script that would increment the counter:
 
 [source,js]
 --------------------------------------------------
-POST test/type1/1/_update
+POST test/_doc/1/_update
 {
     "script" : {
         "source": "ctx._source.counter += params.count",
@@ -51,7 +51,7 @@ will still add it, since its a list):
 
 [source,js]
 --------------------------------------------------
-POST test/type1/1/_update
+POST test/_doc/1/_update
 {
     "script" : {
         "source": "ctx._source.tags.add(params.tag)",
@@ -73,7 +73,7 @@ We can also add a new field to the document:
 
 [source,js]
 --------------------------------------------------
-POST test/type1/1/_update
+POST test/_doc/1/_update
 {
     "script" : "ctx._source.new_field = 'value_of_new_field'"
 }
@@ -85,7 +85,7 @@ Or remove a field from the document:
 
 [source,js]
 --------------------------------------------------
-POST test/type1/1/_update
+POST test/_doc/1/_update
 {
     "script" : "ctx._source.remove('new_field')"
 }
@@ -99,7 +99,7 @@ the doc if the `tags` field contain `green`, otherwise it does nothing
 
 [source,js]
 --------------------------------------------------
-POST test/type1/1/_update
+POST test/_doc/1/_update
 {
     "script" : {
         "source": "if (ctx._source.tags.contains(params.tag)) { ctx.op = 'delete' } else { ctx.op = 'none' }",
@@ -123,7 +123,7 @@ example:
 
 [source,js]
 --------------------------------------------------
-POST test/type1/1/_update
+POST test/_doc/1/_update
 {
     "doc" : {
         "name" : "new_name"
@@ -144,7 +144,7 @@ By default updates that don't change anything detect that they don't change anyt
 
 [source,js]
 --------------------------------------------------
-POST test/type1/1/_update
+POST test/_doc/1/_update
 {
     "doc" : {
         "name" : "new_name"
@@ -167,7 +167,7 @@ the request was ignored.
         "failed": 0
    },
    "_index": "test",
-   "_type": "type1",
+   "_type": "_doc",
    "_id": "1",
    "_version": 6,
    "result": "noop"
@@ -179,7 +179,7 @@ You can disable this behavior by setting "detect_noop": false like this:
 
 [source,js]
 --------------------------------------------------
-POST test/type1/1/_update
+POST test/_doc/1/_update
 {
     "doc" : {
         "name" : "new_name"
@@ -200,7 +200,7 @@ will be inserted as a new document.  If the document does exist, then the
 
 [source,js]
 --------------------------------------------------
-POST test/type1/1/_update
+POST test/_doc/1/_update
 {
     "script" : {
         "source": "ctx._source.counter += params.count",
@@ -255,7 +255,7 @@ value:
 
 [source,js]
 --------------------------------------------------
-POST test/type1/1/_update
+POST test/_doc/1/_update
 {
     "doc" : {
         "name" : "new_name"
diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
index e8c617e9be4..2ebe8c03865 100755
--- a/docs/reference/getting-started.asciidoc
+++ b/docs/reference/getting-started.asciidoc
@@ -380,7 +380,7 @@ Let's now put something into our customer index. We'll index a simple customer d
 
 [source,js]
 --------------------------------------------------
-PUT /customer/doc/1?pretty
+PUT /customer/_doc/1?pretty
 {
   "name": "John Doe"
 }
@@ -393,7 +393,7 @@ And the response:
 --------------------------------------------------
 {
   "_index" : "customer",
-  "_type" : "doc",
+  "_type" : "_doc",
   "_id" : "1",
   "_version" : 1,
   "result" : "created",
@@ -416,7 +416,7 @@ Let's now retrieve that document that we just indexed:
 
 [source,js]
 --------------------------------------------------
-GET /customer/doc/1?pretty
+GET /customer/_doc/1?pretty
 --------------------------------------------------
 // CONSOLE
 // TEST[continued]
@@ -427,7 +427,7 @@ And the response:
 --------------------------------------------------
 {
   "_index" : "customer",
-  "_type" : "doc",
+  "_type" : "_doc",
   "_id" : "1",
   "_version" : 1,
   "found" : true,
@@ -465,11 +465,11 @@ Before we move on, let's take a closer look again at some of the API commands th
 [source,js]
 --------------------------------------------------
 PUT /customer
-PUT /customer/doc/1
+PUT /customer/_doc/1
 {
   "name": "John Doe"
 }
-GET /customer/doc/1
+GET /customer/_doc/1
 DELETE /customer
 --------------------------------------------------
 // CONSOLE
@@ -495,7 +495,7 @@ We've previously seen how we can index a single document. Let's recall that comm
 
 [source,js]
 --------------------------------------------------
-PUT /customer/doc/1?pretty
+PUT /customer/_doc/1?pretty
 {
   "name": "John Doe"
 }
@@ -506,7 +506,7 @@ Again, the above will index the specified document into the customer index, with
 
 [source,js]
 --------------------------------------------------
-PUT /customer/doc/1?pretty
+PUT /customer/_doc/1?pretty
 {
   "name": "Jane Doe"
 }
@@ -518,7 +518,7 @@ The above changes the name of the document with the ID of 1 from "John Doe" to "
 
 [source,js]
 --------------------------------------------------
-PUT /customer/doc/2?pretty
+PUT /customer/_doc/2?pretty
 {
   "name": "Jane Doe"
 }
@@ -534,7 +534,7 @@ This example shows how to index a document without an explicit ID:
 
 [source,js]
 --------------------------------------------------
-POST /customer/doc?pretty
+POST /customer/_doc?pretty
 {
   "name": "Jane Doe"
 }
@@ -552,7 +552,7 @@ This example shows how to update our previous document (ID of 1) by changing the
 
 [source,js]
 --------------------------------------------------
-POST /customer/doc/1/_update?pretty
+POST /customer/_doc/1/_update?pretty
 {
   "doc": { "name": "Jane Doe" }
 }
@@ -564,7 +564,7 @@ This example shows how to update our previous document (ID of 1) by changing the
 
 [source,js]
 --------------------------------------------------
-POST /customer/doc/1/_update?pretty
+POST /customer/_doc/1/_update?pretty
 {
   "doc": { "name": "Jane Doe", "age": 20 }
 }
@@ -576,7 +576,7 @@ Updates can also be performed by using simple scripts. This example uses a scrip
 
 [source,js]
 --------------------------------------------------
-POST /customer/doc/1/_update?pretty
+POST /customer/_doc/1/_update?pretty
 {
   "script" : "ctx._source.age += 5"
 }
@@ -594,7 +594,7 @@ Deleting a document is fairly straightforward. This example shows how to delete
 
 [source,js]
 --------------------------------------------------
-DELETE /customer/doc/2?pretty
+DELETE /customer/_doc/2?pretty
 --------------------------------------------------
 // CONSOLE
 // TEST[continued]
@@ -611,7 +611,7 @@ As a quick example, the following call indexes two documents (ID 1 - John Doe an
 
 [source,js]
 --------------------------------------------------
-POST /customer/doc/_bulk?pretty
+POST /customer/_doc/_bulk?pretty
 {"index":{"_id":"1"}}
 {"name": "John Doe" }
 {"index":{"_id":"2"}}
@@ -623,7 +623,7 @@ This example updates the first document (ID of 1) and then deletes the second do
 
 [source,sh]
 --------------------------------------------------
-POST /customer/doc/_bulk?pretty
+POST /customer/_doc/_bulk?pretty
 {"update":{"_id":"1"}}
 {"doc": { "name": "John Doe becomes Jane Doe" } }
 {"delete":{"_id":"2"}}
@@ -696,7 +696,7 @@ yellow open   bank  l7sSYV2cQXmu6_4rJWVIww   5   1       1000            0    12
 // TESTRESPONSE[s/128.6kb/\\d+(\\.\\d+)?[mk]?b/]
 // TESTRESPONSE[s/l7sSYV2cQXmu6_4rJWVIww/.+/ _cat]
 
-Which means that we just successfully bulk indexed 1000 documents into the bank index (under the account type).
+Which means that we just successfully bulk indexed 1000 documents into the bank index (under the `_doc` type).
 
 === The Search API
 
@@ -731,14 +731,14 @@ And the response (partially shown):
     "max_score" : null,
     "hits" : [ {
       "_index" : "bank",
-      "_type" : "account",
+      "_type" : "_doc",
       "_id" : "0",
       "sort": [0],
       "_score" : null,
       "_source" : {"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO"}
     }, {
       "_index" : "bank",
-      "_type" : "account",
+      "_type" : "_doc",
       "_id" : "1",
       "sort": [1],
       "_score" : null,
@@ -799,14 +799,14 @@ to clutter the docs with it:
     "max_score": null,
     "hits" : [ {
       "_index" : "bank",
-      "_type" : "account",
+      "_type" : "_doc",
       "_id" : "0",
       "sort": [0],
       "_score": null,
       "_source" : {"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO"}
     }, {
       "_index" : "bank",
-      "_type" : "account",
+      "_type" : "_doc",
       "_id" : "1",
       "sort": [1],
       "_score": null,
diff --git a/docs/reference/how-to/recipes/stemming.asciidoc b/docs/reference/how-to/recipes/stemming.asciidoc
index 49d9f8bafa3..4e12dfd7eca 100644
--- a/docs/reference/how-to/recipes/stemming.asciidoc
+++ b/docs/reference/how-to/recipes/stemming.asciidoc
@@ -24,7 +24,7 @@ PUT index
     }
   },
   "mappings": {
-    "type": {
+    "_doc": {
       "properties": {
         "body": {
           "type": "text",
@@ -41,12 +41,12 @@ PUT index
   }
 }
 
-PUT index/type/1
+PUT index/_doc/1
 {
   "body": "Ski resort"
 }
 
-PUT index/type/2
+PUT index/_doc/2
 {
   "body": "A pair of skis"
 }
@@ -89,7 +89,7 @@ GET index/_search
     "hits": [
       {
         "_index": "index",
-        "_type": "type",
+        "_type": "_doc",
         "_id": "2",
         "_score": 0.2876821,
         "_source": {
@@ -98,7 +98,7 @@ GET index/_search
       },
       {
         "_index": "index",
-        "_type": "type",
+        "_type": "_doc",
         "_id": "1",
         "_score": 0.2876821,
         "_source": {
@@ -147,7 +147,7 @@ GET index/_search
     "hits": [
       {
         "_index": "index",
-        "_type": "type",
+        "_type": "_doc",
         "_id": "1",
         "_score": 0.2876821,
         "_source": {
@@ -204,7 +204,7 @@ GET index/_search
     "hits": [
       {
         "_index": "index",
-        "_type": "type",
+        "_type": "_doc",
         "_id": "1",
         "_score": 0.2876821,
         "_source": {
diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc
index db84acd3516..b2fb0a4c2af 100644
--- a/docs/reference/how-to/search-speed.asciidoc
+++ b/docs/reference/how-to/search-speed.asciidoc
@@ -49,7 +49,7 @@ For instance, if documents look like:
 
 [source,js]
 --------------------------------------------------
-PUT index/type/1
+PUT index/_doc/1
 {
   "designation": "spoon",
   "price": 13
@@ -88,7 +88,7 @@ should be mapped as a <>:
 PUT index
 {
   "mappings": {
-    "type": {
+    "_doc": {
       "properties": {
         "price_range": {
           "type": "keyword"
@@ -98,7 +98,7 @@ PUT index
   }
 }
 
-PUT index/type/1
+PUT index/_doc/1
 {
   "designation": "spoon",
   "price": 13,
@@ -152,7 +152,7 @@ For instance the below query:
 
 [source,js]
 --------------------------------------------------
-PUT index/type/1
+PUT index/_doc/1
 {
   "my_date": "2016-05-11T16:30:55.328Z"
 }
@@ -284,7 +284,7 @@ eagerly at refresh-time by configuring mappings as described below:
 PUT index
 {
   "mappings": {
-    "type": {
+    "_doc": {
       "properties": {
         "foo": {
           "type": "keyword",
diff --git a/docs/reference/index-modules/index-sorting.asciidoc b/docs/reference/index-modules/index-sorting.asciidoc
index 3947b2501c2..9d1dfcb1a75 100644
--- a/docs/reference/index-modules/index-sorting.asciidoc
+++ b/docs/reference/index-modules/index-sorting.asciidoc
@@ -25,7 +25,7 @@ PUT twitter
         }
     },
     "mappings": {
-        "tweet": {
+        "_doc": {
             "properties": {
                 "date": {
                     "type": "date"
@@ -53,7 +53,7 @@ PUT twitter
         }
     },
     "mappings": {
-        "tweet": {
+        "_doc": {
             "properties": {
                 "username": {
                     "type": "keyword",
diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc
index 20bf9a51357..85ca9e0cea3 100644
--- a/docs/reference/index-modules/similarity.asciidoc
+++ b/docs/reference/index-modules/similarity.asciidoc
@@ -44,7 +44,7 @@ Here we configure the DFRSimilarity so it can be referenced as
 
 [source,js]
 --------------------------------------------------
-PUT /index/_mapping/book
+PUT /index/_mapping/_doc
 {
   "properties" : {
     "title" : { "type" : "text", "similarity" : "my_similarity" }
@@ -197,7 +197,7 @@ PUT /index
     }
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "field": {
           "type": "text",
@@ -208,12 +208,12 @@ PUT /index
   }
 }
 
-PUT /index/doc/1
+PUT /index/_doc/1
 {
   "field": "foo bar foo"
 }
 
-PUT /index/doc/2
+PUT /index/_doc/2
 {
   "field": "bar baz"
 }
@@ -253,7 +253,7 @@ Which yields:
         "_shard": "[index][0]",
         "_node": "OzrdjxNtQGaqs4DmioFw9A",
         "_index": "index",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 1.9508477,
         "_source": {
@@ -355,7 +355,7 @@ PUT /index
     }
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "field": {
           "type": "text",
@@ -372,12 +372,12 @@ PUT /index
 
 [source,js]
 --------------------------------------------------
-PUT /index/doc/1
+PUT /index/_doc/1
 {
   "field": "foo bar foo"
 }
 
-PUT /index/doc/2
+PUT /index/_doc/2
 {
   "field": "bar baz"
 }
@@ -416,7 +416,7 @@ GET /index/_search?explain=true
         "_shard": "[index][0]",
         "_node": "OzrdjxNtQGaqs4DmioFw9A",
         "_index": "index",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 1.9508477,
         "_source": {
diff --git a/docs/reference/indices/get-field-mapping.asciidoc b/docs/reference/indices/get-field-mapping.asciidoc
index d2f0d5a7508..94df9a9c490 100644
--- a/docs/reference/indices/get-field-mapping.asciidoc
+++ b/docs/reference/indices/get-field-mapping.asciidoc
@@ -12,7 +12,7 @@ For example, consider the following mapping:
 PUT publications
 {
     "mappings": {
-        "article": {
+        "_doc": {
             "properties": {
                 "id": { "type": "text" },
                 "title":  { "type": "text"},
@@ -35,7 +35,7 @@ The following returns the mapping of the field `title` only:
 
 [source,js]
 --------------------------------------------------
-GET publications/_mapping/article/field/title
+GET publications/_mapping/_doc/field/title
 --------------------------------------------------
 // CONSOLE
 
@@ -46,7 +46,7 @@ For which the response is:
 {
    "publications": {
       "mappings": {
-         "article": {
+         "_doc": {
             "title": {
                "full_name": "title",
                "mapping": {
@@ -76,9 +76,9 @@ following are some examples:
 --------------------------------------------------
 GET /twitter,kimchy/_mapping/field/message
 
-GET /_all/_mapping/tweet,book/field/message,user.id
+GET /_all/_mapping/_doc,tweet,book/field/message,user.id
 
-GET /_all/_mapping/tw*/field/*.id
+GET /_all/_mapping/_do*/field/*.id
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -93,7 +93,7 @@ For instance to select the `id` of the `author` field, you must use its full nam
 
 [source,js]
 --------------------------------------------------
-GET publications/_mapping/article/field/author.id,abstract,name
+GET publications/_mapping/_doc/field/author.id,abstract,name
 --------------------------------------------------
 // CONSOLE
 
@@ -104,7 +104,7 @@ returns:
 {
    "publications": {
       "mappings": {
-         "article": {
+         "_doc": {
             "author.id": {
                "full_name": "author.id",
                "mapping": {
@@ -132,7 +132,7 @@ The get field mapping API also supports wildcard notation.
 
 [source,js]
 --------------------------------------------------
-GET publications/_mapping/article/field/a*
+GET publications/_mapping/_doc/field/a*
 --------------------------------------------------
 // CONSOLE
 
@@ -143,7 +143,7 @@ returns:
 {
    "publications": {
       "mappings": {
-         "article": {
+         "_doc": {
             "author.name": {
                "full_name": "author.name",
                "mapping": {
diff --git a/docs/reference/indices/get-mapping.asciidoc b/docs/reference/indices/get-mapping.asciidoc
index d1e45b2dbb0..953f9522a41 100644
--- a/docs/reference/indices/get-mapping.asciidoc
+++ b/docs/reference/indices/get-mapping.asciidoc
@@ -6,7 +6,7 @@ index/type.
 
 [source,js]
 --------------------------------------------------
-GET /twitter/_mapping/tweet
+GET /twitter/_mapping/_doc
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -23,9 +23,9 @@ following are some examples:
 
 [source,js]
 --------------------------------------------------
-GET /_mapping/tweet
+GET /_mapping/_doc
 
-GET /_all/_mapping/tweet
+GET /_all/_mapping/_doc
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc
index a1bc8428557..44a689e98ed 100644
--- a/docs/reference/indices/put-mapping.asciidoc
+++ b/docs/reference/indices/put-mapping.asciidoc
@@ -9,7 +9,7 @@ fields to an existing type:
 PUT twitter <1>
 {}
 
-PUT twitter/_mapping/user <2>
+PUT twitter/_mapping/_doc <2>
 {
   "properties": {
     "name": {
@@ -18,7 +18,7 @@ PUT twitter/_mapping/user <2>
   }
 }
 
-PUT twitter/_mapping/user <3>
+PUT twitter/_mapping/_doc <3>
 {
   "properties": {
     "email": {
@@ -48,7 +48,7 @@ PUT twitter-1
 PUT twitter-2
 
 # Update both mappings
-PUT /twitter-1,twitter-2/_mapping/my_type <1>
+PUT /twitter-1,twitter-2/_mapping/_doc <1>
 {
   "properties": {
     "user_name": {
@@ -84,7 +84,7 @@ For example:
 PUT my_index <1>
 {
   "mappings": {
-    "user": {
+    "_doc": {
       "properties": {
         "name": {
           "properties": {
@@ -101,7 +101,7 @@ PUT my_index <1>
   }
 }
 
-PUT my_index/_mapping/user
+PUT my_index/_mapping/_doc
 {
   "properties": {
     "name": {
diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index 33bb09a1ef6..1e3a361f1b1 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -106,7 +106,7 @@ PUT /%3Clogs-%7Bnow%2Fd%7D-1%3E <1>
   }
 }
 
-PUT logs_write/log/1
+PUT logs_write/_doc/1
 {
   "message": "a dummy log"
 }
diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc
index 463e47dfde9..da1164930bc 100644
--- a/docs/reference/ingest.asciidoc
+++ b/docs/reference/ingest.asciidoc
@@ -26,7 +26,7 @@ tell the ingest node which pipeline to use. For example:
 
 [source,js]
 --------------------------------------------------
-PUT my-index/my-type/my-id?pipeline=my_pipeline_id
+PUT my-index/_doc/my-id?pipeline=my_pipeline_id
 {
   "foo": "bar"
 }
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 54d1a00d335..01e695b4233 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -250,7 +250,7 @@ POST _ingest/pipeline/_simulate
   "docs": [
     {
       "_index": "index",
-      "_type": "type",
+      "_type": "_doc",
       "_id": "id",
       "_source": {
         "foo": "bar"
@@ -258,7 +258,7 @@ POST _ingest/pipeline/_simulate
     },
     {
       "_index": "index",
-      "_type": "type",
+      "_type": "_doc",
       "_id": "id",
       "_source": {
         "foo": "rab"
@@ -279,7 +279,7 @@ Response:
          "doc": {
             "_id": "id",
             "_index": "index",
-            "_type": "type",
+            "_type": "_doc",
             "_source": {
                "field2": "_value",
                "foo": "bar"
@@ -293,7 +293,7 @@ Response:
          "doc": {
             "_id": "id",
             "_index": "index",
-            "_type": "type",
+            "_type": "_doc",
             "_source": {
                "field2": "_value",
                "foo": "rab"
@@ -343,7 +343,7 @@ POST _ingest/pipeline/_simulate?verbose
   "docs": [
     {
       "_index": "index",
-      "_type": "type",
+      "_type": "_doc",
       "_id": "id",
       "_source": {
         "foo": "bar"
@@ -351,7 +351,7 @@ POST _ingest/pipeline/_simulate?verbose
     },
     {
       "_index": "index",
-      "_type": "type",
+      "_type": "_doc",
       "_id": "id",
       "_source": {
         "foo": "rab"
@@ -374,7 +374,7 @@ Response:
                "doc": {
                   "_id": "id",
                   "_index": "index",
-                  "_type": "type",
+                  "_type": "_doc",
                   "_source": {
                      "field2": "_value2",
                      "foo": "bar"
@@ -388,7 +388,7 @@ Response:
                "doc": {
                   "_id": "id",
                   "_index": "index",
-                  "_type": "type",
+                  "_type": "_doc",
                   "_source": {
                      "field3": "_value3",
                      "field2": "_value2",
@@ -407,7 +407,7 @@ Response:
                "doc": {
                   "_id": "id",
                   "_index": "index",
-                  "_type": "type",
+                  "_type": "_doc",
                   "_source": {
                      "field2": "_value2",
                      "foo": "rab"
@@ -421,7 +421,7 @@ Response:
                "doc": {
                   "_id": "id",
                   "_index": "index",
-                  "_type": "type",
+                  "_type": "_doc",
                   "_source": {
                      "field3": "_value3",
                      "field2": "_value2",
@@ -917,7 +917,7 @@ Using that pipeline for an index request:
 
 [source,js]
 --------------------------------------------------
-PUT /myindex/type/1?pipeline=monthlyindex
+PUT /myindex/_doc/1?pipeline=monthlyindex
 {
   "date1" : "2016-04-25T12:02:01.789Z"
 }
@@ -929,7 +929,7 @@ PUT /myindex/type/1?pipeline=monthlyindex
 --------------------------------------------------
 {
   "_index" : "myindex-2016-04-01",
-  "_type" : "type",
+  "_type" : "_doc",
   "_id" : "1",
   "_version" : 1,
   "result" : "created",
@@ -1824,11 +1824,11 @@ was provided in the original index request:
 --------------------------------------------------
 PUT _ingest/pipeline/my_index
 {
-    "description": "use index:my_index and type:my_type",
+    "description": "use index:my_index and type:_doc",
     "processors": [
       {
         "script": {
-          "source": " ctx._index = 'my_index'; ctx._type = 'my_type' "
+          "source": " ctx._index = 'my_index'; ctx._type = '_doc' "
         }
       }
     ]
@@ -1840,7 +1840,7 @@ Using the above pipeline, we can attempt to index a document into the `any_index
 
 [source,js]
 --------------------------------------------------
-PUT any_index/any_type/1?pipeline=my_index
+PUT any_index/_doc/1?pipeline=my_index
 {
   "message": "text"
 }
@@ -1854,7 +1854,7 @@ The response from the above index request:
 --------------------------------------------------
 {
   "_index": "my_index",
-  "_type": "my_type",
+  "_type": "_doc",
   "_id": "1",
   "_version": 1,
   "result": "created",
diff --git a/docs/reference/mapping/dynamic-mapping.asciidoc b/docs/reference/mapping/dynamic-mapping.asciidoc
index b4b9198b173..2066c8b3b03 100644
--- a/docs/reference/mapping/dynamic-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic-mapping.asciidoc
@@ -9,11 +9,11 @@ type, and fields will spring to life automatically:
 
 [source,js]
 --------------------------------------------------
-PUT data/counters/1 <1>
+PUT data/_doc/1 <1>
 { "count": 5 }
 --------------------------------------------------
 // CONSOLE
-<1> Creates the `data` index, the `counters` mapping type, and a field
+<1> Creates the `data` index, the `_doc` mapping type, and a field
     called `count` with datatype `long`.
 
 The automatic detection and addition of new fields is called
diff --git a/docs/reference/mapping/dynamic/field-mapping.asciidoc b/docs/reference/mapping/dynamic/field-mapping.asciidoc
index 2b0afa1562d..8b2e106bfbf 100644
--- a/docs/reference/mapping/dynamic/field-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic/field-mapping.asciidoc
@@ -48,7 +48,7 @@ For example:
 
 [source,js]
 --------------------------------------------------
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "create_date": "2015/09/02"
 }
@@ -69,13 +69,13 @@ Dynamic date detection can be disabled by setting `date_detection` to `false`:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "date_detection": false
     }
   }
 }
 
-PUT my_index/my_type/1 <1>
+PUT my_index/_doc/1 <1>
 {
   "create": "2015/09/02"
 }
@@ -94,13 +94,13 @@ own <>:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "dynamic_date_formats": ["MM/dd/yyyy"]
     }
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "create_date": "09/25/2015"
 }
@@ -122,13 +122,13 @@ correct solution is to map these fields explicitly, but numeric detection
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "numeric_detection": true
     }
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "my_float":   "1.0", <1>
   "my_integer": "1" <2>
diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc
index 90d4ae1de56..c715ee68f8b 100644
--- a/docs/reference/mapping/dynamic/templates.asciidoc
+++ b/docs/reference/mapping/dynamic/templates.asciidoc
@@ -61,7 +61,7 @@ could use the following template:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "dynamic_templates": [
         {
           "integers": {
@@ -90,7 +90,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "my_integer": 5, <1>
   "my_string": "Some string" <2>
@@ -117,7 +117,7 @@ fields:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "dynamic_templates": [
         {
           "longs_as_strings": {
@@ -134,7 +134,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "long_num": "5", <1>
   "long_text": "foo" <2>
@@ -173,7 +173,7 @@ top-level `full_name` field, except for the `middle` field:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "dynamic_templates": [
         {
           "full_name": {
@@ -190,7 +190,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "name": {
     "first":  "Alice",
@@ -214,7 +214,7 @@ field, and disables <> for all non-string fields:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "dynamic_templates": [
         {
           "named_analyzers": {
@@ -240,7 +240,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "english": "Some English text", <1>
   "count":   5 <2>
@@ -268,7 +268,7 @@ you will have to search on the exact same value that was indexed.
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "dynamic_templates": [
         {
           "strings_as_keywords": {
@@ -298,7 +298,7 @@ before 5.0):
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "dynamic_templates": [
         {
           "strings_as_text": {
@@ -326,7 +326,7 @@ disable the storage of these scoring factors in the index and save some space.
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "dynamic_templates": [
         {
           "strings_as_keywords": {
@@ -367,7 +367,7 @@ maybe gain some indexing speed:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "dynamic_templates": [
         {
           "unindexed_longs": {
diff --git a/docs/reference/mapping/fields/field-names-field.asciidoc b/docs/reference/mapping/fields/field-names-field.asciidoc
index 9dd1f17cbb3..2539bf4287a 100644
--- a/docs/reference/mapping/fields/field-names-field.asciidoc
+++ b/docs/reference/mapping/fields/field-names-field.asciidoc
@@ -11,12 +11,12 @@ The value of the `_field_names` field is accessible in queries:
 [source,js]
 --------------------------
 # Example documents
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "title": "This is a document"
 }
 
-PUT my_index/my_type/2?refresh=true
+PUT my_index/_doc/2?refresh=true
 {
   "title": "This is another document",
   "body": "This document has a body"
@@ -48,7 +48,7 @@ disable this field if you want to optimize for indexing speed and do not need
 PUT tweets
 {
   "mappings": {
-    "tweet": {
+    "_doc": {
       "_field_names": {
         "enabled": false
       }
diff --git a/docs/reference/mapping/fields/id-field.asciidoc b/docs/reference/mapping/fields/id-field.asciidoc
index ebfc829d46d..c46ca28af06 100644
--- a/docs/reference/mapping/fields/id-field.asciidoc
+++ b/docs/reference/mapping/fields/id-field.asciidoc
@@ -15,12 +15,12 @@ The value of the `_id` field is accessible in certain queries (`term`,
 [source,js]
 --------------------------
 # Example documents
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "text": "Document with ID 1"
 }
 
-PUT my_index/my_type/2&refresh=true
+PUT my_index/_doc/2&refresh=true
 {
   "text": "Document with ID 2"
 }
diff --git a/docs/reference/mapping/fields/index-field.asciidoc b/docs/reference/mapping/fields/index-field.asciidoc
index 1cd6b05a93d..b11c1cee144 100644
--- a/docs/reference/mapping/fields/index-field.asciidoc
+++ b/docs/reference/mapping/fields/index-field.asciidoc
@@ -16,12 +16,12 @@ but it does not support `prefix`, `wildcard`, `regexp`, or `fuzzy` queries.
 [source,js]
 --------------------------
 # Example documents
-PUT index_1/my_type/1
+PUT index_1/_doc/1
 {
   "text": "Document in index 1"
 }
 
-PUT index_2/my_type/2?refresh=true
+PUT index_2/_doc/2?refresh=true
 {
   "text": "Document in index 2"
 }
diff --git a/docs/reference/mapping/fields/routing-field.asciidoc b/docs/reference/mapping/fields/routing-field.asciidoc
index 5fd8545dece..07fbb80bcb6 100644
--- a/docs/reference/mapping/fields/routing-field.asciidoc
+++ b/docs/reference/mapping/fields/routing-field.asciidoc
@@ -13,12 +13,12 @@ value per document.  For instance:
 
 [source,js]
 ------------------------------
-PUT my_index/my_type/1?routing=user1&refresh=true <1>
+PUT my_index/_doc/1?routing=user1&refresh=true <1>
 {
   "title": "This is a document"
 }
 
-GET my_index/my_type/1?routing=user1 <2>
+GET my_index/_doc/1?routing=user1 <2>
 ------------------------------
 // CONSOLE
 // TESTSETUP
@@ -82,7 +82,7 @@ custom `routing` value required for all CRUD operations:
 PUT my_index2
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "_routing": {
         "required": true <1>
       }
@@ -90,14 +90,14 @@ PUT my_index2
   }
 }
 
-PUT my_index2/my_type/1 <2>
+PUT my_index2/_doc/1 <2>
 {
   "text": "No routing value provided"
 }
 ------------------------------
 // CONSOLE
 // TEST[catch:bad_request]
-<1> Routing is required for `my_type` documents.
+<1> Routing is required for `_doc` documents.
 <2> This index request throws a `routing_missing_exception`.
 
 ==== Unique IDs with custom routing
diff --git a/docs/reference/mapping/fields/source-field.asciidoc b/docs/reference/mapping/fields/source-field.asciidoc
index ea9ed5d29b3..e872d31e563 100644
--- a/docs/reference/mapping/fields/source-field.asciidoc
+++ b/docs/reference/mapping/fields/source-field.asciidoc
@@ -16,7 +16,7 @@ within the index. For this reason, it can be disabled as follows:
 PUT tweets
 {
   "mappings": {
-    "tweet": {
+    "_doc": {
       "_source": {
         "enabled": false
       }
@@ -88,7 +88,7 @@ as follows:
 PUT logs
 {
   "mappings": {
-    "event": {
+    "_doc": {
       "_source": {
         "includes": [
           "*.count",
@@ -103,7 +103,7 @@ PUT logs
   }
 }
 
-PUT logs/event/1
+PUT logs/_doc/1
 {
   "requests": {
     "count": 10,
@@ -119,7 +119,7 @@ PUT logs/event/1
   }
 }
 
-GET logs/event/_search
+GET logs/_search
 {
   "query": {
     "match": {
diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc
index d5b09824599..c1cfa797102 100644
--- a/docs/reference/mapping/fields/type-field.asciidoc
+++ b/docs/reference/mapping/fields/type-field.asciidoc
@@ -14,7 +14,7 @@ scripts, and when sorting:
 --------------------------
 # Example documents
 
-PUT my_index/doc/1?refresh=true
+PUT my_index/_doc/1?refresh=true
 {
   "text": "Document with type 'doc'"
 }
@@ -23,7 +23,7 @@ GET my_index/_search
 {
   "query": {
     "term": {
-      "_type": "doc"  <1>
+      "_type": "_doc"  <1>
     }
   },
   "aggs": {
diff --git a/docs/reference/mapping/fields/uid-field.asciidoc b/docs/reference/mapping/fields/uid-field.asciidoc
index 68b5c82ab0f..2ca3b69a5ae 100644
--- a/docs/reference/mapping/fields/uid-field.asciidoc
+++ b/docs/reference/mapping/fields/uid-field.asciidoc
@@ -13,12 +13,12 @@ and when sorting:
 [source,js]
 --------------------------
 # Example documents
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "text": "Document with ID 1"
 }
 
-PUT my_index/my_type/2?refresh=true
+PUT my_index/_doc/2?refresh=true
 {
   "text": "Document with ID 2"
 }
@@ -31,7 +31,7 @@ GET my_index/_search
 {
   "query": {
     "terms": {
-      "_uid": [ "my_type#1", "my_type#2" ] <1>
+      "_uid": [ "_doc#1", "_doc#2" ] <1>
     }
   },
   "aggs": {
diff --git a/docs/reference/mapping/params/analyzer.asciidoc b/docs/reference/mapping/params/analyzer.asciidoc
index 358be142373..c885b99ab63 100644
--- a/docs/reference/mapping/params/analyzer.asciidoc
+++ b/docs/reference/mapping/params/analyzer.asciidoc
@@ -44,7 +44,7 @@ in the field mapping, as follows:
 PUT /my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "text": { <1>
           "type": "text",
@@ -123,7 +123,7 @@ PUT my_index
       }
    },
    "mappings":{
-      "my_type":{
+      "_doc":{
          "properties":{
             "title": {
                "type":"text",
@@ -136,17 +136,17 @@ PUT my_index
    }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
    "title":"The Quick Brown Fox"
 }
 
-PUT my_index/my_type/2
+PUT my_index/_doc/2
 {
    "title":"A Quick Brown Fox"
 }
 
-GET my_index/my_type/_search
+GET my_index/_search
 {
    "query":{
       "query_string":{
diff --git a/docs/reference/mapping/params/boost.asciidoc b/docs/reference/mapping/params/boost.asciidoc
index 1f9d83dbe50..f2a5e48483c 100644
--- a/docs/reference/mapping/params/boost.asciidoc
+++ b/docs/reference/mapping/params/boost.asciidoc
@@ -9,7 +9,7 @@ Individual fields can be _boosted_ automatically -- count more towards the relev
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "title": {
           "type": "text",
diff --git a/docs/reference/mapping/params/coerce.asciidoc b/docs/reference/mapping/params/coerce.asciidoc
index d3e158185b6..fff0d27b6ff 100644
--- a/docs/reference/mapping/params/coerce.asciidoc
+++ b/docs/reference/mapping/params/coerce.asciidoc
@@ -20,7 +20,7 @@ For instance:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "number_one": {
           "type": "integer"
@@ -34,12 +34,12 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "number_one": "10" <1>
 }
 
-PUT my_index/my_type/2
+PUT my_index/_doc/2
 {
   "number_two": "10" <2>
 }
@@ -67,7 +67,7 @@ PUT my_index
     "index.mapping.coerce": false
   },
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "number_one": {
           "type": "integer",
@@ -81,10 +81,10 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 { "number_one": "10" } <1>
 
-PUT my_index/my_type/2
+PUT my_index/_doc/2
 { "number_two": "10" } <2>
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/mapping/params/copy-to.asciidoc b/docs/reference/mapping/params/copy-to.asciidoc
index 599e93aa12e..447d7baf274 100644
--- a/docs/reference/mapping/params/copy-to.asciidoc
+++ b/docs/reference/mapping/params/copy-to.asciidoc
@@ -11,7 +11,7 @@ the `full_name` field as follows:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "first_name": {
           "type": "text",
@@ -29,7 +29,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "first_name": "John",
   "last_name": "Smith"
diff --git a/docs/reference/mapping/params/doc-values.asciidoc b/docs/reference/mapping/params/doc-values.asciidoc
index afe325310c7..88bc944e3c5 100644
--- a/docs/reference/mapping/params/doc-values.asciidoc
+++ b/docs/reference/mapping/params/doc-values.asciidoc
@@ -26,7 +26,7 @@ value from a script, you can disable doc values in order to save disk space:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "status_code": { <1>
           "type":       "keyword"
diff --git a/docs/reference/mapping/params/dynamic.asciidoc b/docs/reference/mapping/params/dynamic.asciidoc
index 15352842df0..63303abdefd 100644
--- a/docs/reference/mapping/params/dynamic.asciidoc
+++ b/docs/reference/mapping/params/dynamic.asciidoc
@@ -7,7 +7,7 @@ containing the new field.  For instance:
 
 [source,js]
 --------------------------------------------------
-PUT my_index/my_type/1 <1>
+PUT my_index/_doc/1 <1>
 {
   "username": "johnsmith",
   "name": {
@@ -18,7 +18,7 @@ PUT my_index/my_type/1 <1>
 
 GET my_index/_mapping <2>
 
-PUT my_index/my_type/2 <3>
+PUT my_index/_doc/2 <3>
 {
   "username": "marywhite",
   "email": "mary@white.com",
@@ -61,7 +61,7 @@ object or from the mapping type.  For instance:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "dynamic": false, <1>
       "properties": {
         "user": { <2>
diff --git a/docs/reference/mapping/params/eager-global-ordinals.asciidoc b/docs/reference/mapping/params/eager-global-ordinals.asciidoc
index 7e668621cd9..1b4cee65e61 100644
--- a/docs/reference/mapping/params/eager-global-ordinals.asciidoc
+++ b/docs/reference/mapping/params/eager-global-ordinals.asciidoc
@@ -36,7 +36,7 @@ aggregations:
 
 [source,js]
 ------------
-PUT my_index/_mapping/my_type
+PUT my_index/_mapping/_doc
 {
   "properties": {
     "tags": {
@@ -59,7 +59,7 @@ time:
 
 [source,js]
 ------------
-PUT my_index/_mapping/my_type
+PUT my_index/_mapping/_doc
 {
   "properties": {
     "tags": {
diff --git a/docs/reference/mapping/params/enabled.asciidoc b/docs/reference/mapping/params/enabled.asciidoc
index e14e727d6cb..111d431cd25 100644
--- a/docs/reference/mapping/params/enabled.asciidoc
+++ b/docs/reference/mapping/params/enabled.asciidoc
@@ -18,7 +18,7 @@ in any other way:
 PUT my_index
 {
   "mappings": {
-    "session": {
+    "_doc": {
       "properties": {
         "user_id": {
           "type":  "keyword"
@@ -34,7 +34,7 @@ PUT my_index
   }
 }
 
-PUT my_index/session/session_1
+PUT my_index/_doc/session_1
 {
   "user_id": "kimchy",
   "session_data": { <2>
@@ -45,7 +45,7 @@ PUT my_index/session/session_1
   "last_updated": "2015-12-06T18:20:22"
 }
 
-PUT my_index/session/session_2
+PUT my_index/_doc/session_2
 {
   "user_id": "jpountz",
   "session_data": "none", <3>
@@ -66,13 +66,13 @@ retrieved, but none of its contents are indexed in any way:
 PUT my_index
 {
   "mappings": {
-    "session": { <1>
+    "_doc": { <1>
       "enabled": false
     }
   }
 }
 
-PUT my_index/session/session_1
+PUT my_index/_doc/session_1
 {
   "user_id": "kimchy",
   "session_data": {
@@ -83,12 +83,12 @@ PUT my_index/session/session_1
   "last_updated": "2015-12-06T18:20:22"
 }
 
-GET my_index/session/session_1 <2>
+GET my_index/_doc/session_1 <2>
 
 GET my_index/_mapping <3>
 --------------------------------------------------
 // CONSOLE
-<1> The entire `session` mapping type is disabled.
+<1> The entire `_doc` mapping type is disabled.
 <2> The document can be retrieved.
 <3> Checking the mapping reveals that no fields have been added.
 
diff --git a/docs/reference/mapping/params/fielddata.asciidoc b/docs/reference/mapping/params/fielddata.asciidoc
index 0ba05fdf396..19899c76d8a 100644
--- a/docs/reference/mapping/params/fielddata.asciidoc
+++ b/docs/reference/mapping/params/fielddata.asciidoc
@@ -58,7 +58,7 @@ enabled for aggregations, as follows:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "my_field": { <1>
           "type": "text",
@@ -84,7 +84,7 @@ You can enable fielddata on an existing `text` field using the
 
 [source,js]
 -----------------------------------
-PUT my_index/_mapping/my_type
+PUT my_index/_mapping/_doc
 {
   "properties": {
     "my_field": { <1>
@@ -126,7 +126,7 @@ number of docs that the segment should contain with `min_segment_size`:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "tag": {
           "type": "text",
diff --git a/docs/reference/mapping/params/format.asciidoc b/docs/reference/mapping/params/format.asciidoc
index 530807589c4..85cad16cb53 100644
--- a/docs/reference/mapping/params/format.asciidoc
+++ b/docs/reference/mapping/params/format.asciidoc
@@ -14,7 +14,7 @@ Besides the <>, your own
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "date": {
           "type":   "date",
diff --git a/docs/reference/mapping/params/ignore-above.asciidoc b/docs/reference/mapping/params/ignore-above.asciidoc
index 2db12a33368..95704c6c8bb 100644
--- a/docs/reference/mapping/params/ignore-above.asciidoc
+++ b/docs/reference/mapping/params/ignore-above.asciidoc
@@ -8,7 +8,7 @@ Strings longer than the `ignore_above` setting will not be indexed or stored.
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "message": {
           "type": "keyword",
@@ -19,12 +19,12 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1 <2>
+PUT my_index/_doc/1 <2>
 {
   "message": "Syntax error"
 }
 
-PUT my_index/my_type/2 <3>
+PUT my_index/_doc/2 <3>
 {
   "message": "Syntax error with some long stacktrace"
 }
diff --git a/docs/reference/mapping/params/ignore-malformed.asciidoc b/docs/reference/mapping/params/ignore-malformed.asciidoc
index 905a0f7d78a..9a2fef1a23e 100644
--- a/docs/reference/mapping/params/ignore-malformed.asciidoc
+++ b/docs/reference/mapping/params/ignore-malformed.asciidoc
@@ -17,7 +17,7 @@ For example:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "number_one": {
           "type": "integer",
@@ -31,13 +31,13 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "text":       "Some text value",
   "number_one": "foo" <1>
 }
 
-PUT my_index/my_type/2
+PUT my_index/_doc/2
 {
   "text":       "Some text value",
   "number_two": "foo" <2>
@@ -67,7 +67,7 @@ PUT my_index
     "index.mapping.ignore_malformed": true <1>
   },
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "number_one": { <1>
           "type": "byte"
diff --git a/docs/reference/mapping/params/index-options.asciidoc b/docs/reference/mapping/params/index-options.asciidoc
index e2cd6ce20e1..9bbdd017e02 100644
--- a/docs/reference/mapping/params/index-options.asciidoc
+++ b/docs/reference/mapping/params/index-options.asciidoc
@@ -38,7 +38,7 @@ all other fields use `docs` as the default.
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "text": {
           "type": "text",
@@ -49,7 +49,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "text": "Quick brown fox"
 }
diff --git a/docs/reference/mapping/params/multi-fields.asciidoc b/docs/reference/mapping/params/multi-fields.asciidoc
index 80a92a5d83e..1489304bbaa 100644
--- a/docs/reference/mapping/params/multi-fields.asciidoc
+++ b/docs/reference/mapping/params/multi-fields.asciidoc
@@ -11,7 +11,7 @@ search, and as a `keyword` field for sorting or aggregations:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "city": {
           "type": "text",
@@ -26,12 +26,12 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "city": "New York"
 }
 
-PUT my_index/my_type/2
+PUT my_index/_doc/2
 {
   "city": "York"
 }
@@ -79,7 +79,7 @@ which stems words into their root form:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "text": { <1>
           "type": "text",
@@ -95,10 +95,10 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 { "text": "quick brown fox" } <3>
 
-PUT my_index/my_type/2
+PUT my_index/_doc/2
 { "text": "quick brown foxes" } <3>
 
 GET my_index/_search
diff --git a/docs/reference/mapping/params/normalizer.asciidoc b/docs/reference/mapping/params/normalizer.asciidoc
index 44a27dab758..c69e816380d 100644
--- a/docs/reference/mapping/params/normalizer.asciidoc
+++ b/docs/reference/mapping/params/normalizer.asciidoc
@@ -25,7 +25,7 @@ PUT index
     }
   },
   "mappings": {
-    "type": {
+    "_doc": {
       "properties": {
         "foo": {
           "type": "keyword",
@@ -36,17 +36,17 @@ PUT index
   }
 }
 
-PUT index/type/1
+PUT index/_doc/1
 {
   "foo": "BÀR"
 }
 
-PUT index/type/2
+PUT index/_doc/2
 {
   "foo": "bar"
 }
 
-PUT index/type/3
+PUT index/_doc/3
 {
   "foo": "baz"
 }
@@ -84,7 +84,7 @@ both index and query time.
     "hits": [
       {
         "_index": "index",
-        "_type": "type",
+        "_type": "_doc",
         "_id": "2",
         "_score": 0.2876821,
         "_source": {
@@ -93,7 +93,7 @@ both index and query time.
       },
       {
         "_index": "index",
-        "_type": "type",
+        "_type": "_doc",
         "_id": "1",
         "_score": 0.2876821,
         "_source": {
diff --git a/docs/reference/mapping/params/norms.asciidoc b/docs/reference/mapping/params/norms.asciidoc
index 0d259c0bc7c..842f0af64b5 100644
--- a/docs/reference/mapping/params/norms.asciidoc
+++ b/docs/reference/mapping/params/norms.asciidoc
@@ -20,7 +20,7 @@ Norms can be disabled (but not reenabled) after the fact, using the
 
 [source,js]
 ------------
-PUT my_index/_mapping/my_type
+PUT my_index/_mapping/_doc
 {
   "properties": {
     "title": {
diff --git a/docs/reference/mapping/params/null-value.asciidoc b/docs/reference/mapping/params/null-value.asciidoc
index ee1ea0c3257..f7a780d8662 100644
--- a/docs/reference/mapping/params/null-value.asciidoc
+++ b/docs/reference/mapping/params/null-value.asciidoc
@@ -13,7 +13,7 @@ the specified value so that it can be indexed and searched.  For instance:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "status_code": {
           "type":       "keyword",
@@ -24,12 +24,12 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "status_code": null
 }
 
-PUT my_index/my_type/2
+PUT my_index/_doc/2
 {
   "status_code": [] <2>
 }
diff --git a/docs/reference/mapping/params/position-increment-gap.asciidoc b/docs/reference/mapping/params/position-increment-gap.asciidoc
index addd8aadbbd..c002d5a861a 100644
--- a/docs/reference/mapping/params/position-increment-gap.asciidoc
+++ b/docs/reference/mapping/params/position-increment-gap.asciidoc
@@ -13,12 +13,12 @@ For example:
 
 [source,js]
 --------------------------------------------------
-PUT my_index/groups/1
+PUT my_index/_doc/1
 {
     "names": [ "John Abraham", "Lincoln Smith"]
 }
 
-GET my_index/groups/_search
+GET my_index/_search
 {
     "query": {
         "match_phrase": {
@@ -29,7 +29,7 @@ GET my_index/groups/_search
     }
 }
 
-GET my_index/groups/_search
+GET my_index/_search
 {
     "query": {
         "match_phrase": {
@@ -54,7 +54,7 @@ The `position_increment_gap` can be specified in the mapping.  For instance:
 PUT my_index
 {
   "mappings": {
-    "groups": {
+    "_doc": {
       "properties": {
         "names": {
           "type": "text",
@@ -65,12 +65,12 @@ PUT my_index
   }
 }
 
-PUT my_index/groups/1
+PUT my_index/_doc/1
 {
     "names": [ "John Abraham", "Lincoln Smith"]
 }
 
-GET my_index/groups/_search
+GET my_index/_search
 {
     "query": {
         "match_phrase": {
diff --git a/docs/reference/mapping/params/properties.asciidoc b/docs/reference/mapping/params/properties.asciidoc
index 9efe1f8e82c..fa74bffd9d3 100644
--- a/docs/reference/mapping/params/properties.asciidoc
+++ b/docs/reference/mapping/params/properties.asciidoc
@@ -18,7 +18,7 @@ field, and a `nested` field:
 PUT my_index
 {
   "mappings": {
-    "my_type": { <1>
+    "_doc": { <1>
       "properties": {
         "manager": { <2>
           "properties": {
@@ -38,7 +38,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1 <4>
+PUT my_index/_doc/1 <4>
 {
   "region": "US",
   "manager": {
@@ -58,7 +58,7 @@ PUT my_index/my_type/1 <4>
 }
 --------------------------------------------------
 // CONSOLE
-<1> Properties under the `my_type` mapping type.
+<1> Properties under the `_doc` mapping type.
 <2> Properties under the `manager` object field.
 <3> Properties under the `employees` nested field.
 <4> An example document which corresponds to the above mapping.
diff --git a/docs/reference/mapping/params/search-analyzer.asciidoc b/docs/reference/mapping/params/search-analyzer.asciidoc
index 26cda7afa18..1a67dfab631 100644
--- a/docs/reference/mapping/params/search-analyzer.asciidoc
+++ b/docs/reference/mapping/params/search-analyzer.asciidoc
@@ -38,7 +38,7 @@ PUT my_index
     }
   },
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "text": {
           "type": "text",
@@ -50,7 +50,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "text": "Quick Brown Fox" <3>
 }
diff --git a/docs/reference/mapping/params/similarity.asciidoc b/docs/reference/mapping/params/similarity.asciidoc
index 0a5979c9d32..3509cd0cf8e 100644
--- a/docs/reference/mapping/params/similarity.asciidoc
+++ b/docs/reference/mapping/params/similarity.asciidoc
@@ -39,7 +39,7 @@ as follows:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "default_field": { <1>
           "type": "text"
diff --git a/docs/reference/mapping/params/store.asciidoc b/docs/reference/mapping/params/store.asciidoc
index 53cac7493ff..56c62385b59 100644
--- a/docs/reference/mapping/params/store.asciidoc
+++ b/docs/reference/mapping/params/store.asciidoc
@@ -21,7 +21,7 @@ to extract those fields from a large `_source` field:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "title": {
           "type": "text",
@@ -39,7 +39,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "title":   "Some short title",
   "date":    "2015-01-01",
diff --git a/docs/reference/mapping/params/term-vector.asciidoc b/docs/reference/mapping/params/term-vector.asciidoc
index 121ab39a6df..c71c23f8a48 100644
--- a/docs/reference/mapping/params/term-vector.asciidoc
+++ b/docs/reference/mapping/params/term-vector.asciidoc
@@ -32,7 +32,7 @@ index.
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "text": {
           "type":        "text",
@@ -43,7 +43,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "text": "Quick brown fox"
 }
diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc
index 17874f2d3ba..070d189a0ff 100644
--- a/docs/reference/mapping/removal_of_types.asciidoc
+++ b/docs/reference/mapping/removal_of_types.asciidoc
@@ -164,7 +164,7 @@ You could achieve the same thing by adding a custom `type` field as follows:
 PUT twitter
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "type": { "type": "keyword" }, <1>
         "name": { "type": "text" },
@@ -177,7 +177,7 @@ PUT twitter
   }
 }
 
-PUT twitter/doc/user-kimchy
+PUT twitter/_doc/user-kimchy
 {
   "type": "user", <1>
   "name": "Shay Banon",
@@ -185,7 +185,7 @@ PUT twitter/doc/user-kimchy
   "email": "shay@kimchy.com"
 }
 
-PUT twitter/doc/tweet-1
+PUT twitter/_doc/tweet-1
 {
   "type": "tweet", <1>
   "user_name": "kimchy",
@@ -244,7 +244,9 @@ Elasticsearch 6.x::
 * Indices created in 5.x will continue to function in 6.x as they did in 5.x.
 
 * Indices created in 6.x only allow a single-type per index.  Any name
-  can be used for the type, but there can be only one.
+  can be used for the type, but there can be only one. The preferred type name
+  is `_doc`, so that index APIs have the same path as they will have in 7.0:
+  `PUT {index}/_doc/{id}` and `POST {index}/_doc`
 
 * The `_type` name can no longer be combined with the `_id` to form the `_uid`
   field. The `_uid` field has become an alias for the `_id` field.
@@ -257,7 +259,9 @@ Elasticsearch 6.x::
 Elasticsearch 7.x::
 
 * The `type` parameter in URLs are optional.  For instance, indexing
-  a document no longer requires a document `type`.
+  a document no longer requires a document `type`.  The new index APIs
+  are `PUT {index}/_doc/{id}` in case of explicit ids and `POST {index}/_doc`
+  for auto-generated ids.
 
 * The `GET|PUT _mapping` APIs support a query string parameter
   (`include_type_name`) which indicates whether the body should include
@@ -298,7 +302,7 @@ PUT users
     "index.mapping.single_type": true
   },
   "mappings": {
-    "user": {
+    "_doc": {
       "properties": {
         "name": {
           "type": "text"
@@ -320,7 +324,7 @@ PUT tweets
     "index.mapping.single_type": true
   },
   "mappings": {
-    "tweet": {
+    "_doc": {
       "properties": {
         "content": {
           "type": "text"
@@ -372,7 +376,7 @@ documents of different types which have conflicting IDs:
 PUT new_twitter
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "type": {
           "type": "keyword"
@@ -410,7 +414,7 @@ POST _reindex
     "source": """
       ctx._source.type = ctx._type;
       ctx._id = ctx._type + '-' + ctx._id;
-      ctx._type = 'doc';
+      ctx._type = '_doc';
     """
   }
 }
diff --git a/docs/reference/mapping/types/array.asciidoc b/docs/reference/mapping/types/array.asciidoc
index a1bf9f1eb65..385c61ebcd7 100644
--- a/docs/reference/mapping/types/array.asciidoc
+++ b/docs/reference/mapping/types/array.asciidoc
@@ -40,7 +40,7 @@ are supported out of the box:
 
 [source,js]
 --------------------------------------------------
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "message": "some arrays in this document...",
   "tags":  [ "elasticsearch", "wow" ], <1>
@@ -56,7 +56,7 @@ PUT my_index/my_type/1
   ]
 }
 
-PUT my_index/my_type/2 <3>
+PUT my_index/_doc/2 <3>
 {
   "message": "no arrays in this document...",
   "tags":  "elasticsearch",
diff --git a/docs/reference/mapping/types/binary.asciidoc b/docs/reference/mapping/types/binary.asciidoc
index 0633c29801e..9b7b032d0ab 100644
--- a/docs/reference/mapping/types/binary.asciidoc
+++ b/docs/reference/mapping/types/binary.asciidoc
@@ -10,7 +10,7 @@ stored by default and is not searchable:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "name": {
           "type": "text"
@@ -23,7 +23,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "name": "Some binary blob",
   "blob": "U29tZSBiaW5hcnkgYmxvYg==" <1>
diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc
index 4fe577b9f63..4ae350412be 100644
--- a/docs/reference/mapping/types/boolean.asciidoc
+++ b/docs/reference/mapping/types/boolean.asciidoc
@@ -20,7 +20,7 @@ For example:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "is_published": {
           "type": "boolean"
@@ -30,7 +30,7 @@ PUT my_index
   }
 }
 
-POST my_index/my_type/1
+POST my_index/_doc/1
 {
   "is_published": "true" <1>
 }
@@ -55,12 +55,12 @@ return `1` and `0`:
 
 [source,js]
 --------------------------------------------------
-POST my_index/my_type/1
+POST my_index/_doc/1
 {
   "is_published": true
 }
 
-POST my_index/my_type/2
+POST my_index/_doc/2
 {
   "is_published": false
 }
diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc
index da7798519a3..bdce1d6c46f 100644
--- a/docs/reference/mapping/types/date.asciidoc
+++ b/docs/reference/mapping/types/date.asciidoc
@@ -26,7 +26,7 @@ For instance:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "date": {
           "type": "date" <1>
@@ -36,13 +36,13 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 { "date": "2015-01-01" } <2>
 
-PUT my_index/my_type/2
+PUT my_index/_doc/2
 { "date": "2015-01-01T12:10:30Z" } <3>
 
-PUT my_index/my_type/3
+PUT my_index/_doc/3
 { "date": 1420070400001 } <4>
 
 GET my_index/_search
@@ -70,7 +70,7 @@ into a string.
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "date": {
           "type":   "date",
diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc
index ccfc93e34c5..60d2c8c46da 100644
--- a/docs/reference/mapping/types/geo-point.asciidoc
+++ b/docs/reference/mapping/types/geo-point.asciidoc
@@ -18,7 +18,7 @@ There are four ways that a geo-point may be specified, as demonstrated below:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "location": {
           "type": "geo_point"
@@ -28,7 +28,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "text": "Geo-point as an object",
   "location": { <1>
@@ -37,19 +37,19 @@ PUT my_index/my_type/1
   }
 }
 
-PUT my_index/my_type/2
+PUT my_index/_doc/2
 {
   "text": "Geo-point as a string",
   "location": "41.12,-71.34" <2>
 }
 
-PUT my_index/my_type/3
+PUT my_index/_doc/3
 {
   "text": "Geo-point as a geohash",
   "location": "drm3btev3e86" <3>
 }
 
-PUT my_index/my_type/4
+PUT my_index/_doc/4
 {
   "text": "Geo-point as an array",
   "location": [ -71.34, 41.12 ] <4>
diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc
index afb482c454b..512b0d72545 100644
--- a/docs/reference/mapping/types/ip.asciidoc
+++ b/docs/reference/mapping/types/ip.asciidoc
@@ -9,7 +9,7 @@ https://en.wikipedia.org/wiki/IPv6[IPv6] addresses.
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "ip_addr": {
           "type": "ip"
@@ -19,7 +19,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "ip_addr": "192.168.1.1"
 }
diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
index 821fb055744..92f6167ef2c 100644
--- a/docs/reference/mapping/types/keyword.asciidoc
+++ b/docs/reference/mapping/types/keyword.asciidoc
@@ -18,7 +18,7 @@ Below is an example of a mapping for a keyword field:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "tags": {
           "type":  "keyword"
diff --git a/docs/reference/mapping/types/nested.asciidoc b/docs/reference/mapping/types/nested.asciidoc
index b5d96513cf7..0b8db376e86 100644
--- a/docs/reference/mapping/types/nested.asciidoc
+++ b/docs/reference/mapping/types/nested.asciidoc
@@ -14,7 +14,7 @@ following document:
 
 [source,js]
 --------------------------------------------------
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "group" : "fans",
   "user" : [ <1>
@@ -78,7 +78,7 @@ queried independently of the others, with the <
@@ -88,7 +88,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "group" : "fans",
   "user" : [
diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc
index 757c958fb16..5e9de317bac 100644
--- a/docs/reference/mapping/types/numeric.asciidoc
+++ b/docs/reference/mapping/types/numeric.asciidoc
@@ -20,7 +20,7 @@ Below is an example of configuring a mapping with numeric fields:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "number_of_bytes": {
           "type": "integer"
diff --git a/docs/reference/mapping/types/object.asciidoc b/docs/reference/mapping/types/object.asciidoc
index 31f728d1ae0..8c3dd676393 100644
--- a/docs/reference/mapping/types/object.asciidoc
+++ b/docs/reference/mapping/types/object.asciidoc
@@ -6,7 +6,7 @@ objects which, in turn, may contain inner objects themselves:
 
 [source,js]
 --------------------------------------------------
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 { <1>
   "region": "US",
   "manager": { <2>
@@ -44,7 +44,7 @@ An explicit mapping for the above document could look like this:
 PUT my_index
 {
   "mappings": {
-    "my_type": { <1>
+    "_doc": { <1>
       "properties": {
         "region": {
           "type": "keyword"
diff --git a/docs/reference/mapping/types/parent-join.asciidoc b/docs/reference/mapping/types/parent-join.asciidoc
index f91d045ae0b..055109a4ce2 100644
--- a/docs/reference/mapping/types/parent-join.asciidoc
+++ b/docs/reference/mapping/types/parent-join.asciidoc
@@ -12,7 +12,7 @@ A parent/child relation can be defined as follows:
 PUT my_index
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "my_join_field": { <1>
           "type": "join",
@@ -36,7 +36,7 @@ For instance the following example creates two `parent` documents in the `questi
 
 [source,js]
 --------------------------------------------------
-PUT my_index/doc/1?refresh
+PUT my_index/_doc/1?refresh
 {
   "text": "This is a question",
   "my_join_field": {
@@ -44,7 +44,7 @@ PUT my_index/doc/1?refresh
   }
 }
 
-PUT my_index/doc/2?refresh
+PUT my_index/_doc/2?refresh
 {
   "text": "This is a another question",
   "my_join_field": {
@@ -62,13 +62,13 @@ as a shortcut instead of encapsulating it in the normal object notation:
 
 [source,js]
 --------------------------------------------------
-PUT my_index/doc/1?refresh
+PUT my_index/_doc/1?refresh
 {
   "text": "This is a question",
   "my_join_field": "question" <1>
 }
 
-PUT my_index/doc/2?refresh
+PUT my_index/_doc/2?refresh
 {
   "text": "This is another question",
   "my_join_field": "question"
@@ -89,7 +89,7 @@ For instance the following example shows how to index two `child` documents:
 
 [source,js]
 --------------------------------------------------
-PUT my_index/doc/3?routing=1&refresh <1>
+PUT my_index/_doc/3?routing=1&refresh <1>
 {
   "text": "This is an answer",
   "my_join_field": {
@@ -98,7 +98,7 @@ PUT my_index/doc/3?routing=1&refresh <1>
   }
 }
 
-PUT my_index/doc/4?routing=1&refresh
+PUT my_index/_doc/4?routing=1&refresh
 {
   "text": "This is another answer",
   "my_join_field": {
@@ -180,7 +180,7 @@ Will return:
         "hits": [
             {
                 "_index": "my_index",
-                "_type": "doc",
+                "_type": "_doc",
                 "_id": "1",
                 "_score": null,
                 "_source": {
@@ -193,7 +193,7 @@ Will return:
             },
             {
                 "_index": "my_index",
-                "_type": "doc",
+                "_type": "_doc",
                 "_id": "2",
                 "_score": null,
                 "_source": {
@@ -206,7 +206,7 @@ Will return:
             },
             {
                 "_index": "my_index",
-                "_type": "doc",
+                "_type": "_doc",
                 "_id": "3",
                 "_score": null,
                 "_routing": "1",
@@ -223,7 +223,7 @@ Will return:
             },
             {
                 "_index": "my_index",
-                "_type": "doc",
+                "_type": "_doc",
                 "_id": "4",
                 "_score": null,
                 "_routing": "1",
@@ -319,7 +319,7 @@ make sense to disable eager loading:
 PUT my_index
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "my_join_field": {
           "type": "join",
@@ -358,7 +358,7 @@ It is also possible to define multiple children for a single parent:
 PUT my_index
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "my_join_field": {
           "type": "join",
@@ -388,7 +388,7 @@ Multiple levels of parent/child:
 PUT my_index
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "my_join_field": {
           "type": "join",
@@ -423,7 +423,7 @@ to the grand-parent (the greater parent of the lineage):
 
 [source,js]
 --------------------------------------------------
-PUT my_index/doc/3?routing=1&refresh <1>
+PUT my_index/_doc/3?routing=1&refresh <1>
 {
   "text": "This is a vote",
   "my_join_field": {
diff --git a/docs/reference/mapping/types/percolator.asciidoc b/docs/reference/mapping/types/percolator.asciidoc
index f5590692ca5..35a4d7d6b75 100644
--- a/docs/reference/mapping/types/percolator.asciidoc
+++ b/docs/reference/mapping/types/percolator.asciidoc
@@ -18,7 +18,7 @@ If the following mapping configures the `percolator` field type for the
 PUT my_index
 {
     "mappings": {
-        "doc": {
+        "_doc": {
             "properties": {
                 "query": {
                     "type": "percolator"
@@ -38,7 +38,7 @@ Then you can index a query:
 
 [source,js]
 --------------------------------------------------
-PUT my_index/doc/match_value
+PUT my_index/_doc/match_value
 {
     "query" : {
         "match" : {
@@ -73,7 +73,7 @@ Lets take a look at the following index with a percolator field type:
 PUT index
 {
   "mappings": {
-    "doc" : {
+    "_doc" : {
       "properties": {
         "query" : {
           "type" : "percolator"
@@ -98,7 +98,7 @@ POST _aliases
   ]
 }
 
-PUT queries/doc/1?refresh
+PUT queries/_doc/1?refresh
 {
   "query" : {
     "match" : {
@@ -121,7 +121,7 @@ to read your queries you need to reindex your queries into a new index on the cu
 PUT new_index
 {
   "mappings": {
-    "doc" : {
+    "_doc" : {
       "properties": {
         "query" : {
           "type" : "percolator"
@@ -205,7 +205,7 @@ now returns matches from the new index:
     "hits": [
       {
         "_index": "new_index", <1>
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 0.2876821,
         "_source": {
@@ -280,7 +280,7 @@ PUT /test_index
     }
   },
   "mappings": {
-    "doc" : {
+    "_doc" : {
       "properties": {
         "query" : {
           "type": "percolator"
@@ -341,7 +341,7 @@ All the tokens in the returned order need to replace the query text in the perco
 
 [source,js]
 --------------------------------------------------
-PUT /test_index/doc/1?refresh
+PUT /test_index/_doc/1?refresh
 {
   "query" : {
     "match" : {
@@ -400,7 +400,7 @@ This results in a response like this:
     "hits": [
       {
         "_index": "test_index",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 0.2876821,
         "_source": {
diff --git a/docs/reference/mapping/types/range.asciidoc b/docs/reference/mapping/types/range.asciidoc
index 3fdd7576a17..76aba682771 100644
--- a/docs/reference/mapping/types/range.asciidoc
+++ b/docs/reference/mapping/types/range.asciidoc
@@ -19,7 +19,7 @@ Below is an example of configuring a mapping with various range fields followed
 PUT range_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "expected_attendees": {
           "type": "integer_range"
@@ -33,7 +33,7 @@ PUT range_index
   }
 }
 
-PUT range_index/my_type/1
+PUT range_index/_doc/1
 {
   "expected_attendees" : { <2>
     "gte" : 10,
@@ -86,7 +86,7 @@ The result produced by the above query.
     "hits" : [
       {
         "_index" : "range_index",
-        "_type" : "my_type",
+        "_type" : "_doc",
         "_id" : "1",
         "_score" : 1.0,
         "_source" : {
diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc
index 2f0d2dbce46..4be9c00a710 100644
--- a/docs/reference/mapping/types/text.asciidoc
+++ b/docs/reference/mapping/types/text.asciidoc
@@ -20,7 +20,7 @@ Below is an example of a mapping for a text field:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "full_name": {
           "type":  "text"
diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc
index 60a95ec1d40..da4220f4bb4 100644
--- a/docs/reference/mapping/types/token-count.asciidoc
+++ b/docs/reference/mapping/types/token-count.asciidoc
@@ -12,7 +12,7 @@ For instance:
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "name": { <1>
           "type": "text",
@@ -28,10 +28,10 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 { "name": "John Smith" }
 
-PUT my_index/my_type/2
+PUT my_index/_doc/2
 { "name": "Rachel Alice Williams" }
 
 GET my_index/_search
diff --git a/docs/reference/modules/cross-cluster-search.asciidoc b/docs/reference/modules/cross-cluster-search.asciidoc
index 6bdbffdfe88..0d54aa7e756 100644
--- a/docs/reference/modules/cross-cluster-search.asciidoc
+++ b/docs/reference/modules/cross-cluster-search.asciidoc
@@ -168,7 +168,7 @@ must be prefixed with the cluster alias separated by a `:` character:
 
 [source,js]
 --------------------------------------------------
-GET /cluster_one:twitter/tweet/_search
+GET /cluster_one:twitter/_search
 {
   "query": {
     "match": {
@@ -203,7 +203,7 @@ GET /cluster_one:twitter/tweet/_search
     "hits": [
       {
         "_index": "cluster_one:twitter",
-        "_type": "tweet",
+        "_type": "_doc",
         "_id": "0",
         "_score": 1,
         "_source": {
@@ -227,7 +227,7 @@ clusters:
 
 [source,js]
 --------------------------------------------------
-GET /cluster_one:twitter,twitter/tweet/_search
+GET /cluster_one:twitter,twitter/_search
 {
   "query": {
     "match": {
@@ -266,7 +266,7 @@ will be prefixed with their remote cluster name:
     "hits": [
       {
         "_index": "cluster_one:twitter",
-        "_type": "tweet",
+        "_type": "_doc",
         "_id": "0",
         "_score": 1,
         "_source": {
@@ -278,7 +278,7 @@ will be prefixed with their remote cluster name:
       },
       {
         "_index": "twitter",
-        "_type": "tweet",
+        "_type": "_doc",
         "_id": "0",
         "_score": 2,
         "_source": {
@@ -320,7 +320,7 @@ PUT _cluster/settings
 
 [source,js]
 --------------------------------------------------
-GET /cluster_one:twitter,cluster_two:twitter,twitter/tweet/_search <1>
+GET /cluster_one:twitter,cluster_two:twitter,twitter/_search <1>
 {
   "query": {
     "match": {
@@ -355,7 +355,7 @@ GET /cluster_one:twitter,cluster_two:twitter,twitter/tweet/_search <1>
     "hits": [
       {
         "_index": "cluster_one:twitter",
-        "_type": "tweet",
+        "_type": "_doc",
         "_id": "0",
         "_score": 1,
         "_source": {
@@ -367,7 +367,7 @@ GET /cluster_one:twitter,cluster_two:twitter,twitter/tweet/_search <1>
       },
       {
         "_index": "twitter",
-        "_type": "tweet",
+        "_type": "_doc",
         "_id": "0",
         "_score": 2,
         "_source": {
diff --git a/docs/reference/modules/scripting/fields.asciidoc b/docs/reference/modules/scripting/fields.asciidoc
index afb5055a4dd..1225d16333f 100644
--- a/docs/reference/modules/scripting/fields.asciidoc
+++ b/docs/reference/modules/scripting/fields.asciidoc
@@ -44,13 +44,13 @@ relevance `_score` of each document:
 
 [source,js]
 -------------------------------------
-PUT my_index/my_type/1?refresh
+PUT my_index/_doc/1?refresh
 {
   "text": "quick brown fox",
   "popularity": 1
 }
 
-PUT my_index/my_type/2?refresh
+PUT my_index/_doc/2?refresh
 {
   "text": "quick fox",
   "popularity": 5
@@ -89,7 +89,7 @@ store, enabled by default on all fields except for <
           "type": "text"
@@ -182,7 +182,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1?refresh
+PUT my_index/_doc/1?refresh
 {
   "title": "Mr",
   "first_name": "Barry",
diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc
index 0393472e684..725808e1936 100644
--- a/docs/reference/modules/scripting/using.asciidoc
+++ b/docs/reference/modules/scripting/using.asciidoc
@@ -22,7 +22,7 @@ For example, the following script is used in a search request to return a
 
 [source,js]
 -------------------------------------
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "my_field": 5
 }
diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
index 6fce9487f33..e8db949bbc6 100644
--- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
+++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
@@ -9,7 +9,7 @@ bounding box. Assuming the following indexed document:
 PUT /my_locations
 {
     "mappings": {
-        "location": {
+        "_doc": {
             "properties": {
                 "pin": {
                     "properties": {
@@ -23,7 +23,7 @@ PUT /my_locations
     }
 }
 
-PUT /my_locations/location/1
+PUT /my_locations/_doc/1
 {
     "pin" : {
         "location" : {
diff --git a/docs/reference/query-dsl/geo-distance-query.asciidoc b/docs/reference/query-dsl/geo-distance-query.asciidoc
index e961ed811ea..2bf24c942cb 100644
--- a/docs/reference/query-dsl/geo-distance-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-query.asciidoc
@@ -10,7 +10,7 @@ document:
 PUT /my_locations
 {
     "mappings": {
-        "location": {
+        "_doc": {
             "properties": {
                 "pin": {
                     "properties": {
@@ -24,7 +24,7 @@ PUT /my_locations
     }
 }
 
-PUT /my_locations/location/1
+PUT /my_locations/_doc/1
 {
     "pin" : {
         "location" : {
@@ -43,7 +43,7 @@ filter:
 
 [source,js]
 --------------------------------------------------
-GET /my_locations/location/_search
+GET /my_locations/_search
 {
     "query": {
         "bool" : {
@@ -76,7 +76,7 @@ representations of the geo point, the filter can accept it as well:
 
 [source,js]
 --------------------------------------------------
-GET /my_locations/location/_search
+GET /my_locations/_search
 {
     "query": {
         "bool" : {
@@ -106,7 +106,7 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
-GET /my_locations/location/_search
+GET /my_locations/_search
 {
     "query": {
         "bool" : {
@@ -133,7 +133,7 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
-GET /my_locations/location/_search
+GET /my_locations/_search
 {
     "query": {
         "bool" : {
@@ -157,7 +157,7 @@ GET /my_locations/location/_search
 
 [source,js]
 --------------------------------------------------
-GET /my_locations/location/_search
+GET /my_locations/_search
 {
     "query": {
         "bool" : {
diff --git a/docs/reference/query-dsl/geo-shape-query.asciidoc b/docs/reference/query-dsl/geo-shape-query.asciidoc
index 00fd3b5609b..08b504951e1 100644
--- a/docs/reference/query-dsl/geo-shape-query.asciidoc
+++ b/docs/reference/query-dsl/geo-shape-query.asciidoc
@@ -27,7 +27,7 @@ Given the following index:
 PUT /example
 {
     "mappings": {
-        "doc": {
+        "_doc": {
             "properties": {
                 "location": {
                     "type": "geo_shape"
@@ -37,7 +37,7 @@ PUT /example
     }
 }
 
-POST /example/doc?refresh
+POST /example/_doc?refresh
 {
     "name": "Wind & Wetter, Berlin, Germany",
     "location": {
@@ -102,7 +102,7 @@ shape:
 PUT /shapes
 {
     "mappings": {
-        "doc": {
+        "_doc": {
             "properties": {
                 "location": {
                     "type": "geo_shape"
@@ -112,7 +112,7 @@ PUT /shapes
     }
 }
 
-PUT /shapes/doc/deu
+PUT /shapes/_doc/deu
 {
     "location": {
         "type": "envelope",
@@ -129,7 +129,7 @@ GET /example/_search
                     "location": {
                         "indexed_shape": {
                             "index": "shapes",
-                            "type": "doc",
+                            "type": "_doc",
                             "id": "deu",
                             "path": "location"
                         }
diff --git a/docs/reference/query-dsl/ids-query.asciidoc b/docs/reference/query-dsl/ids-query.asciidoc
index 09541ce51d3..5eb52a5dda5 100644
--- a/docs/reference/query-dsl/ids-query.asciidoc
+++ b/docs/reference/query-dsl/ids-query.asciidoc
@@ -10,7 +10,7 @@ GET /_search
 {
     "query": {
         "ids" : {
-            "type" : "my_type",
+            "type" : "_doc",
             "values" : ["1", "4", "100"]
         }
     }
diff --git a/docs/reference/query-dsl/mlt-query.asciidoc b/docs/reference/query-dsl/mlt-query.asciidoc
index 68714b8b7fe..6cb984263e6 100644
--- a/docs/reference/query-dsl/mlt-query.asciidoc
+++ b/docs/reference/query-dsl/mlt-query.asciidoc
@@ -80,7 +80,7 @@ GET /_search
                         "first": "Ben",
                         "last": "Grimm"
                     },
-                    "tweet": "You got no idea what I'd... what I'd give to be invisible."
+                    "_doc": "You got no idea what I'd... what I'd give to be invisible."
                   }
             },
             {
diff --git a/docs/reference/query-dsl/parent-id-query.asciidoc b/docs/reference/query-dsl/parent-id-query.asciidoc
index eaf4a26ba15..aa869235412 100644
--- a/docs/reference/query-dsl/parent-id-query.asciidoc
+++ b/docs/reference/query-dsl/parent-id-query.asciidoc
@@ -9,7 +9,7 @@ Given the following mapping definition:
 PUT my_index
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "my_join_field": {
           "type": "join",
@@ -22,13 +22,13 @@ PUT my_index
   }
 }
 
-PUT my_index/doc/1?refresh
+PUT my_index/_doc/1?refresh
 {
   "text": "This is a parent document",
   "my_join_field": "my_parent"
 }
 
-PUT my_index/doc/2?routing=1&refresh
+PUT my_index/_doc/2?routing=1&refresh
 {
   "text": "This is a child document",
   "my_join_field": {
diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index 410475f42e9..9a97a6bd12e 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -16,7 +16,7 @@ Create an index with two fields:
 PUT /my-index
 {
     "mappings": {
-        "doc": {
+        "_doc": {
             "properties": {
                 "message": {
                     "type": "text"
@@ -44,7 +44,7 @@ Register a query in the percolator:
 
 [source,js]
 --------------------------------------------------
-PUT /my-index/doc/1?refresh
+PUT /my-index/_doc/1?refresh
 {
     "query" : {
         "match" : {
@@ -94,7 +94,7 @@ The above request will yield the following response:
     "hits": [
       { <1>
         "_index": "my-index",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 0.5753642,
         "_source": {
@@ -238,7 +238,7 @@ GET /my-index/_search
     "hits": [
       {
         "_index": "my-index",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 1.5606477,
         "_source": {
@@ -277,7 +277,7 @@ Index the document we want to percolate:
 
 [source,js]
 --------------------------------------------------
-PUT /my-index/doc/2
+PUT /my-index/_doc/2
 {
   "message" : "A new bonsai tree in the office"
 }
@@ -290,7 +290,7 @@ Index response:
 --------------------------------------------------
 {
   "_index": "my-index",
-  "_type": "doc",
+  "_type": "_doc",
   "_id": "2",
   "_version": 1,
   "_shards": {
@@ -315,7 +315,7 @@ GET /my-index/_search
         "percolate" : {
             "field": "query",
             "index" : "my-index",
-            "type" : "doc",
+            "type" : "_doc",
             "id" : "2",
             "version" : 1 <1>
         }
@@ -347,7 +347,7 @@ Save a query:
 
 [source,js]
 --------------------------------------------------
-PUT /my-index/doc/3?refresh
+PUT /my-index/_doc/3?refresh
 {
     "query" : {
         "match" : {
@@ -363,7 +363,7 @@ Save another query:
 
 [source,js]
 --------------------------------------------------
-PUT /my-index/doc/4?refresh
+PUT /my-index/_doc/4?refresh
 {
     "query" : {
         "match" : {
@@ -418,7 +418,7 @@ This will yield the following response.
     "hits": [
       {
         "_index": "my-index",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "3",
         "_score": 0.5753642,
         "_source": {
@@ -439,7 +439,7 @@ This will yield the following response.
       },
       {
         "_index": "my-index",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "4",
         "_score": 0.5753642,
         "_source": {
@@ -523,7 +523,7 @@ The slightly different response:
     "hits": [
       {
         "_index": "my-index",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 1.5606477,
         "_source": {
@@ -619,7 +619,7 @@ The above search request returns a response similar to this:
     "hits": [
       {
         "_index": "my-index",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 0.5753642,
         "_source": {
diff --git a/docs/reference/query-dsl/term-query.asciidoc b/docs/reference/query-dsl/term-query.asciidoc
index bdb62b4d698..4b668203a33 100644
--- a/docs/reference/query-dsl/term-query.asciidoc
+++ b/docs/reference/query-dsl/term-query.asciidoc
@@ -88,7 +88,7 @@ To demonstrate, try out the example below.  First, create an index, specifying t
 PUT my_index
 {
   "mappings": {
-    "my_type": {
+    "_doc": {
       "properties": {
         "full_text": {
           "type":  "text" <1>
@@ -101,7 +101,7 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1
+PUT my_index/_doc/1
 {
   "full_text":   "Quick Foxes!", <3>
   "exact_value": "Quick Foxes!"  <4>
@@ -118,7 +118,7 @@ Now, compare the results for the `term` query and the `match` query:
 
 [source,js]
 --------------------------------------------------
-GET my_index/my_type/_search
+GET my_index/_search
 {
   "query": {
     "term": {
@@ -127,7 +127,7 @@ GET my_index/my_type/_search
   }
 }
 
-GET my_index/my_type/_search
+GET my_index/_search
 {
   "query": {
     "term": {
@@ -136,7 +136,7 @@ GET my_index/my_type/_search
   }
 }
 
-GET my_index/my_type/_search
+GET my_index/_search
 {
   "query": {
     "term": {
@@ -145,7 +145,7 @@ GET my_index/my_type/_search
   }
 }
 
-GET my_index/my_type/_search
+GET my_index/_search
 {
   "query": {
     "match": {
diff --git a/docs/reference/query-dsl/terms-query.asciidoc b/docs/reference/query-dsl/terms-query.asciidoc
index f6d8c9a23cf..5a32ee5b4af 100644
--- a/docs/reference/query-dsl/terms-query.asciidoc
+++ b/docs/reference/query-dsl/terms-query.asciidoc
@@ -65,12 +65,12 @@ all the tweets that match the followers of user 2.
 
 [source,js]
 --------------------------------------------------
-PUT /users/user/2
+PUT /users/_doc/2
 {
     "followers" : ["1", "3"]
 }
 
-PUT /tweets/tweet/1
+PUT /tweets/_doc/1
 {
     "user" : "1"
 }
@@ -96,7 +96,7 @@ inner objects, for example:
 
 [source,js]
 --------------------------------------------------
-PUT /users/user/2
+PUT /users/_doc/2
 {
  "followers" : [
    {
diff --git a/docs/reference/query-dsl/terms-set-query.asciidoc b/docs/reference/query-dsl/terms-set-query.asciidoc
index 659f840cccb..35bc17e1f0f 100644
--- a/docs/reference/query-dsl/terms-set-query.asciidoc
+++ b/docs/reference/query-dsl/terms-set-query.asciidoc
@@ -17,7 +17,7 @@ be a number field:
 PUT /my-index
 {
     "mappings": {
-        "doc": {
+        "_doc": {
             "properties": {
                 "required_matches": {
                     "type": "long"
@@ -27,13 +27,13 @@ PUT /my-index
     }
 }
 
-PUT /my-index/doc/1?refresh
+PUT /my-index/_doc/1?refresh
 {
     "codes": ["ghi", "jkl"],
     "required_matches": 2
 }
 
-PUT /my-index/doc/2?refresh
+PUT /my-index/_doc/2?refresh
 {
     "codes": ["def", "ghi"],
     "required_matches": 2
@@ -79,7 +79,7 @@ Response:
     "hits": [
       {
         "_index": "my-index",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "2",
         "_score": 0.5753642,
         "_source": {
diff --git a/docs/reference/query-dsl/type-query.asciidoc b/docs/reference/query-dsl/type-query.asciidoc
index 05e909bc366..96edda27eff 100644
--- a/docs/reference/query-dsl/type-query.asciidoc
+++ b/docs/reference/query-dsl/type-query.asciidoc
@@ -9,7 +9,7 @@ GET /_search
 {
     "query": {
         "type" : {
-            "value" : "my_type"
+            "value" : "_doc"
         }
     }
 }    
diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc
index 3b15b02dda2..f35f79d42ab 100644
--- a/docs/reference/search.asciidoc
+++ b/docs/reference/search.asciidoc
@@ -34,7 +34,7 @@ only the relevant shard:
 
 [source,js]
 --------------------------------------------------
-POST /twitter/tweet/_search?routing=kimchy
+POST /twitter/_search?routing=kimchy
 {
     "query": {
         "bool" : {
diff --git a/docs/reference/search/count.asciidoc b/docs/reference/search/count.asciidoc
index dbab691867f..44340dfb9c0 100644
--- a/docs/reference/search/count.asciidoc
+++ b/docs/reference/search/count.asciidoc
@@ -10,14 +10,14 @@ body. Here is an example:
 
 [source,js]
 --------------------------------------------------
-PUT /twitter/tweet/1?refresh
+PUT /twitter/_doc/1?refresh
 {
     "user": "kimchy"
 }
 
-GET /twitter/tweet/_count?q=user:kimchy
+GET /twitter/_doc/_count?q=user:kimchy
 
-GET /twitter/tweet/_count
+GET /twitter/_doc/_count
 {
     "query" : {
         "term" : { "user" : "kimchy" }
diff --git a/docs/reference/search/explain.asciidoc b/docs/reference/search/explain.asciidoc
index 7f7024f1e81..fd09984f169 100644
--- a/docs/reference/search/explain.asciidoc
+++ b/docs/reference/search/explain.asciidoc
@@ -15,7 +15,7 @@ Full query example:
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/0/_explain
+GET /twitter/_doc/0/_explain
 {
       "query" : {
         "match" : { "message" : "elasticsearch" }
@@ -31,7 +31,7 @@ This will yield the following result:
 --------------------------------------------------
 {
    "_index": "twitter",
-   "_type": "tweet",
+   "_type": "_doc",
    "_id": "0",
    "matched": true,
    "explanation": {
@@ -104,7 +104,7 @@ explain api:
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/0/_explain?q=message:search
+GET /twitter/_doc/0/_explain?q=message:search
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc
index aa0f36b14ba..b683d73a670 100644
--- a/docs/reference/search/request-body.asciidoc
+++ b/docs/reference/search/request-body.asciidoc
@@ -7,7 +7,7 @@ example:
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/_search
+GET /twitter/_search
 {
     "query" : {
         "term" : { "user" : "kimchy" }
@@ -36,7 +36,7 @@ And here is a sample response:
         "hits" : [
             {
                 "_index" : "twitter",
-                "_type" : "tweet",
+                "_type" : "_doc",
                 "_id" : "0",
                 "_score": 1.3862944,
                 "_source" : {
diff --git a/docs/reference/search/request/collapse.asciidoc b/docs/reference/search/request/collapse.asciidoc
index b4322e36f93..97d85329330 100644
--- a/docs/reference/search/request/collapse.asciidoc
+++ b/docs/reference/search/request/collapse.asciidoc
@@ -7,7 +7,7 @@ For instance the query below retrieves the best tweet for each user and sorts th
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/_search
+GET /twitter/_search
 {
     "query": {
         "match": {
@@ -41,7 +41,7 @@ It is also possible to expand each collapsed top hits with the `inner_hits` opti
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/_search
+GET /twitter/_search
 {
     "query": {
         "match": {
@@ -75,7 +75,7 @@ multiple representations of the collapsed hits.
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/_search
+GET /twitter/_search
 {
     "query": {
         "match": {
diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc
index 066df3e6fa0..bb11a4c9810 100644
--- a/docs/reference/search/request/highlighting.asciidoc
+++ b/docs/reference/search/request/highlighting.asciidoc
@@ -810,7 +810,7 @@ When using the `plain` highlighter, you can choose between the `simple` and
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_search
+GET twitter/_search
 {
     "query" : {
         "match_phrase": { "message": "number 1" }
@@ -842,7 +842,7 @@ Response:
         "hits": [
             {
                 "_index": "twitter",
-                "_type": "tweet",
+                "_type": "_doc",
                 "_id": "1",
                 "_score": 1.601195,
                 "_source": {
@@ -866,7 +866,7 @@ Response:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_search
+GET twitter/_search
 {
     "query" : {
         "match_phrase": { "message": "number 1" }
@@ -898,7 +898,7 @@ Response:
         "hits": [
             {
                 "_index": "twitter",
-                "_type": "tweet",
+                "_type": "_doc",
                 "_id": "1",
                 "_score": 1.601195,
                 "_source": {
diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc
index ca459fe801e..781b9f555a2 100644
--- a/docs/reference/search/request/inner-hits.asciidoc
+++ b/docs/reference/search/request/inner-hits.asciidoc
@@ -87,7 +87,7 @@ The nested `inner_hits` can be used to include nested inner objects as inner hit
 PUT test
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "comments": {
           "type": "nested"
@@ -97,7 +97,7 @@ PUT test
   }
 }
 
-PUT test/doc/1?refresh
+PUT test/_doc/1?refresh
 {
   "title": "Test title",
   "comments": [
@@ -141,7 +141,7 @@ An example of a response snippet that could be generated from the above search r
     "hits": [
       {
         "_index": "test",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 1.0,
         "_source": ...,
@@ -153,7 +153,7 @@ An example of a response snippet that could be generated from the above search r
               "hits": [
                 {
                   "_index": "test",
-                  "_type": "doc",
+                  "_type": "_doc",
                   "_id": "1",
                   "_nested": {
                     "field": "comments",
@@ -207,7 +207,7 @@ including the source and solely rely on doc values fields. Like this:
 PUT test
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "comments": {
           "type": "nested"
@@ -217,7 +217,7 @@ PUT test
   }
 }
 
-PUT test/doc/1?refresh
+PUT test/_doc/1?refresh
 {
   "title": "Test title",
   "comments": [
@@ -264,7 +264,7 @@ Response not included in text but tested for completeness sake.
     "hits": [
       {
         "_index": "test",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 1.0444683,
         "_source": ...,
@@ -276,7 +276,7 @@ Response not included in text but tested for completeness sake.
               "hits": [
                 {
                   "_index": "test",
-                  "_type": "doc",
+                  "_type": "_doc",
                   "_id": "1",
                   "_nested": {
                     "field": "comments",
@@ -315,7 +315,7 @@ with the root hits then the following path can be defined:
 PUT test
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "comments": {
           "type": "nested",
@@ -330,7 +330,7 @@ PUT test
   }
 }
 
-PUT test/doc/1?refresh
+PUT test/_doc/1?refresh
 {
   "title": "Test title",
   "comments": [
@@ -379,7 +379,7 @@ Which would look like:
     "hits": [
       {
         "_index": "test",
-        "_type": "doc",
+        "_type": "_doc",
         "_id": "1",
         "_score": 0.6931472,
         "_source": ...,
@@ -391,7 +391,7 @@ Which would look like:
               "hits": [
                 {
                   "_index": "test",
-                  "_type": "doc",
+                  "_type": "_doc",
                   "_id": "1",
                   "_nested": {
                     "field": "comments",
@@ -431,7 +431,7 @@ The parent/child `inner_hits` can be used to include parent or child:
 PUT test
 {
   "mappings": {
-    "doc": {
+    "_doc": {
       "properties": {
         "my_join_field": {
           "type": "join",
@@ -444,13 +444,13 @@ PUT test
   }
 }
 
-PUT test/doc/1?refresh
+PUT test/_doc/1?refresh
 {
   "number": 1,
   "my_join_field": "my_parent"
 }
 
-PUT test/doc/2?routing=1&refresh
+PUT test/_doc/2?routing=1&refresh
 {
   "number": 1,
   "my_join_field": {
@@ -490,7 +490,7 @@ An example of a response snippet that could be generated from the above search r
         "hits": [
             {
                 "_index": "test",
-                "_type": "doc",
+                "_type": "_doc",
                 "_id": "1",
                 "_score": 1.0,
                 "_source": {
@@ -505,7 +505,7 @@ An example of a response snippet that could be generated from the above search r
                             "hits": [
                                 {
                                     "_index": "test",
-                                    "_type": "doc",
+                                    "_type": "_doc",
                                     "_id": "2",
                                     "_score": 1.0,
                                     "_routing": "1",
diff --git a/docs/reference/search/request/post-filter.asciidoc b/docs/reference/search/request/post-filter.asciidoc
index 249a8e4b1f6..636824bc067 100644
--- a/docs/reference/search/request/post-filter.asciidoc
+++ b/docs/reference/search/request/post-filter.asciidoc
@@ -12,7 +12,7 @@ Imagine that you are selling shirts that have the following properties:
 PUT /shirts
 {
     "mappings": {
-        "item": {
+        "_doc": {
             "properties": {
                 "brand": { "type": "keyword"},
                 "color": { "type": "keyword"},
@@ -22,7 +22,7 @@ PUT /shirts
     }
 }
 
-PUT /shirts/item/1?refresh
+PUT /shirts/_doc/1?refresh
 {
     "brand": "gucci",
     "color": "red",
diff --git a/docs/reference/search/request/scroll.asciidoc b/docs/reference/search/request/scroll.asciidoc
index 91f7b0ec85d..b1166eae9f9 100644
--- a/docs/reference/search/request/scroll.asciidoc
+++ b/docs/reference/search/request/scroll.asciidoc
@@ -38,7 +38,7 @@ should keep the ``search context'' alive (see <>), eg `?s
 
 [source,js]
 --------------------------------------------------
-POST /twitter/tweet/_search?scroll=1m
+POST /twitter/_search?scroll=1m
 {
     "size": 100,
     "query": {
@@ -191,7 +191,7 @@ can be consumed independently:
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/_search?scroll=1m
+GET /twitter/_search?scroll=1m
 {
     "slice": {
         "id": 0, <1>
@@ -203,7 +203,7 @@ GET /twitter/tweet/_search?scroll=1m
         }
     }
 }
-GET /twitter/tweet/_search?scroll=1m
+GET /twitter/_search?scroll=1m
 {
     "slice": {
         "id": 1,
@@ -254,7 +254,7 @@ slice gets deterministic results.
 
 [source,js]
 --------------------------------------------------
-GET /twitter/tweet/_search?scroll=1m
+GET /twitter/_search?scroll=1m
 {
     "slice": {
         "field": "date",
diff --git a/docs/reference/search/request/search-after.asciidoc b/docs/reference/search/request/search-after.asciidoc
index 87f5abba8b2..ed4b35a090a 100644
--- a/docs/reference/search/request/search-after.asciidoc
+++ b/docs/reference/search/request/search-after.asciidoc
@@ -11,7 +11,7 @@ The idea is to use the results from the previous page to help the retrieval of t
 Suppose that the query to retrieve the first page looks like this:
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_search
+GET twitter/_search
 {
     "size": 10,
     "query": {
@@ -39,7 +39,7 @@ For instance we can use the `sort values` of the last document and pass it to `s
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_search
+GET twitter/_search
 {
     "size": 10,
     "query": {
diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc
index 3549e18dbb0..ac1c8388e83 100644
--- a/docs/reference/search/request/sort.asciidoc
+++ b/docs/reference/search/request/sort.asciidoc
@@ -12,7 +12,7 @@ Assuming the following index mapping:
 PUT /my_index
 {
     "mappings": {
-        "my_type": {
+        "_doc": {
             "properties": {
                 "post_date": { "type": "date" },
                 "user": {
@@ -31,7 +31,7 @@ PUT /my_index
 
 [source,js]
 --------------------------------------------------
-GET /my_index/my_type/_search
+GET /my_index/_search
 {
     "sort" : [
         { "post_date" : {"order" : "asc"}},
@@ -92,7 +92,7 @@ the average price per document.
 
 [source,js]
 --------------------------------------------------
-PUT /my_index/my_type/1?refresh
+PUT /my_index/_doc/1?refresh
 {
    "product": "chocolate",
    "price": [20, 4]
diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc
index 77ae8a47dd9..0b6f5fb4d40 100644
--- a/docs/reference/search/search.asciidoc
+++ b/docs/reference/search/search.asciidoc
@@ -36,7 +36,7 @@ We can also search all tweets with a certain tag across several indices
 
 [source,js]
 --------------------------------------------------
-GET /kimchy,elasticsearch/tweet/_search?q=tag:wow
+GET /kimchy,elasticsearch/_search?q=tag:wow
 --------------------------------------------------
 // CONSOLE
 // TEST[s/^/PUT kimchy\nPUT elasticsearch\n/]
@@ -46,7 +46,7 @@ placeholder:
 
 [source,js]
 --------------------------------------------------
-GET /_all/tweet/_search?q=tag:wow
+GET /_all/_search?q=tag:wow
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
diff --git a/docs/reference/search/suggesters/completion-suggest.asciidoc b/docs/reference/search/suggesters/completion-suggest.asciidoc
index 5d8f5fa1cc5..e3101a5dfb4 100644
--- a/docs/reference/search/suggesters/completion-suggest.asciidoc
+++ b/docs/reference/search/suggesters/completion-suggest.asciidoc
@@ -27,7 +27,7 @@ which indexes the field values for fast completions.
 PUT music
 {
     "mappings": {
-        "song" : {
+        "_doc" : {
             "properties" : {
                 "suggest" : {
                     "type" : "completion"
@@ -86,7 +86,7 @@ the suggestions will be scored. Indexing a suggestion is as follows:
 
 [source,js]
 --------------------------------------------------
-PUT music/song/1?refresh
+PUT music/_doc/1?refresh
 {
     "suggest" : {
         "input": [ "Nevermind", "Nirvana" ],
@@ -112,7 +112,7 @@ You can index multiple suggestions for a document as follows:
 
 [source,js]
 --------------------------------------------------
-PUT music/song/1?refresh
+PUT music/_doc/1?refresh
 {
     "suggest" : [
         {
@@ -134,7 +134,7 @@ a weight with suggestion(s) in the shorthand form.
 
 [source,js]
 --------------------------------------------------
-PUT music/song/1?refresh
+PUT music/_doc/1?refresh
 {
   "suggest" : [ "Nevermind", "Nirvana" ]
 }
@@ -193,7 +193,7 @@ returns this response:
       "options" : [ {
         "text" : "Nirvana",
         "_index": "music",
-        "_type": "song",
+        "_type": "_doc",
         "_id": "1",
         "_score": 1.0,
         "_source": {
@@ -269,7 +269,7 @@ Which should look like:
             "options" : [ {
                 "text" : "Nirvana",
                 "_index": "music",
-                "_type": "song",
+                "_type": "_doc",
                 "_id": "1",
                 "_score": 1.0,
                 "_source": {
diff --git a/docs/reference/search/suggesters/context-suggest.asciidoc b/docs/reference/search/suggesters/context-suggest.asciidoc
index 064f919c25d..9226c29b9ad 100644
--- a/docs/reference/search/suggesters/context-suggest.asciidoc
+++ b/docs/reference/search/suggesters/context-suggest.asciidoc
@@ -21,7 +21,7 @@ field:
 PUT place
 {
     "mappings": {
-        "shops" : {
+        "_doc" : {
             "properties" : {
                 "suggest" : {
                     "type" : "completion",
@@ -44,7 +44,7 @@ PUT place
 PUT place_path_category
 {
     "mappings": {
-        "shops" : {
+        "_doc" : {
             "properties" : {
                 "suggest" : {
                     "type" : "completion",
@@ -97,7 +97,7 @@ be sent in the suggest field like this:
 
 [source,js]
 --------------------------------------------------
-PUT place/shops/1
+PUT place/_doc/1
 {
     "suggest": {
         "input": ["timmy's", "starbucks", "dunkin donuts"],
@@ -115,7 +115,7 @@ add the categories:
 
 [source,js]
 --------------------------------------------------
-PUT place_path_category/shops/1
+PUT place_path_category/_doc/1
 {
     "suggest": ["timmy's", "starbucks", "dunkin donuts"],
     "cat": ["cafe", "food"] <1>
@@ -248,7 +248,7 @@ with two geo location contexts:
 
 [source,js]
 --------------------------------------------------
-PUT place/shops/1
+PUT place/_doc/1
 {
     "suggest": {
         "input": "timmy's",
diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc
index 86a57b515ed..108c25a7c3b 100644
--- a/docs/reference/search/uri-request.asciidoc
+++ b/docs/reference/search/uri-request.asciidoc
@@ -8,7 +8,7 @@ example:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_search?q=user:kimchy
+GET twitter/_search?q=user:kimchy
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -32,7 +32,7 @@ And here is a sample response:
         "hits" : [
             {
                 "_index" : "twitter",
-                "_type" : "tweet",
+                "_type" : "_doc",
                 "_id" : "0",
                 "_score": 1.3862944,
                 "_source" : {
diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc
index a9de086cbbb..27b09ad407a 100644
--- a/docs/reference/search/validate.asciidoc
+++ b/docs/reference/search/validate.asciidoc
@@ -6,7 +6,7 @@ without executing it. We'll use the following test data to explain _validate:
 
 [source,js]
 --------------------------------------------------
-PUT twitter/tweet/_bulk?refresh
+PUT twitter/_doc/_bulk?refresh
 {"index":{"_id":1}}
 {"user" : "kimchy", "post_date" : "2009-11-15T14:12:12", "message" : "trying out Elasticsearch"}
 {"index":{"_id":2}}
@@ -60,7 +60,7 @@ The query may also be sent in the request body:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_validate/query
+GET twitter/_doc/_validate/query
 {
   "query" : {
     "bool" : {
@@ -87,7 +87,7 @@ due to dynamic mapping, and 'foo' does not correctly parse into a date:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_validate/query
+GET twitter/_doc/_validate/query
 {
   "query": {
     "query_string": {
@@ -110,7 +110,7 @@ about why a query failed:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_validate/query?explain=true
+GET twitter/_doc/_validate/query?explain=true
 {
   "query": {
     "query_string": {
@@ -150,7 +150,7 @@ For More Like This:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_validate/query?rewrite=true
+GET twitter/_doc/_validate/query?rewrite=true
 {
   "query": {
     "more_like_this": {
@@ -197,7 +197,7 @@ For Fuzzy Queries:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_validate/query?rewrite=true&all_shards=true
+GET twitter/_doc/_validate/query?rewrite=true&all_shards=true
 {
   "query": {
     "match": {
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java
index 258084bdab9..654da9ae67b 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java
@@ -65,7 +65,7 @@ public class RangeFieldQueryStringQueryBuilderTests extends AbstractQueryTestCas
 
     @Override
     protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
-        mapperService.merge("doc", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("doc",
+        mapperService.merge("_doc", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("_doc",
             INTEGER_RANGE_FIELD_NAME, "type=integer_range",
             LONG_RANGE_FIELD_NAME, "type=long_range",
             FLOAT_RANGE_FIELD_NAME, "type=float_range",
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
index 5b392f102ff..885c19c6cd4 100644
--- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
@@ -73,7 +73,7 @@ import static org.hamcrest.CoreMatchers.notNullValue;
 
 public class HasChildQueryBuilderTests extends AbstractQueryTestCase {
 
-    private static final String TYPE = "doc";
+    private static final String TYPE = "_doc";
     private static final String PARENT_DOC = "parent";
     private static final String CHILD_DOC = "child";
 
@@ -97,7 +97,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase {
-    private static final String TYPE = "doc";
+    private static final String TYPE = "_doc";
     private static final String PARENT_DOC = "parent";
     private static final String CHILD_DOC = "child";
 
@@ -81,7 +81,7 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase {
 
-    private static final String TYPE = "doc";
+    private static final String TYPE = "_doc";
     private static final String JOIN_FIELD_NAME = "join_field";
     private static final String PARENT_NAME = "parent";
     private static final String CHILD_NAME = "child";
@@ -73,7 +73,7 @@ public class ParentIdQueryBuilderTests extends AbstractQueryTestCase>
             currentTypes = new String[0]; // no types
             break;
         default:
-            currentTypes = new String[] { "doc" };
+            currentTypes = new String[] { "_doc" };
             break;
         }
         randomTypes = getRandomTypes();

From 1de927c80d851a610b368acf322c1e66a3b69638 Mon Sep 17 00:00:00 2001
From: Jack Conradson 
Date: Thu, 14 Dec 2017 09:08:10 -0800
Subject: [PATCH 267/297] Painless: Clean Up Painless Cast Object (#27794)

Added static methods to make creating Painless casts obvious as to what is
being boxed/unboxed.
---
 .../painless/AnalyzerCaster.java              | 380 +++++++++---------
 .../elasticsearch/painless/Definition.java    |  39 +-
 .../painless/node/NodeToStringTests.java      |   4 +-
 3 files changed, 218 insertions(+), 205 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 02b7105593f..89f358d17e0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -46,421 +46,421 @@ public final class AnalyzerCaster {
 
         if (actual.dynamic) {
             if (expected.clazz == boolean.class) {
-                return new Cast(definition.DefType, definition.BooleanType, explicit, null, definition.booleanType, null, null);
+                return Cast.unboxTo(definition.DefType, definition.BooleanType, explicit, definition.booleanType);
             } else if (expected.clazz == byte.class) {
-                return new Cast(definition.DefType, definition.ByteType, explicit, null, definition.byteType, null, null);
+                return Cast.unboxTo(definition.DefType, definition.ByteType, explicit, definition.byteType);
             } else if (expected.clazz == short.class) {
-                return new Cast(definition.DefType, definition.ShortType, explicit, null, definition.shortType, null, null);
+                return Cast.unboxTo(definition.DefType, definition.ShortType, explicit, definition.shortType);
             } else if (expected.clazz == char.class) {
-                return new Cast(definition.DefType, definition.CharacterType, explicit, null, definition.charType, null, null);
+                return Cast.unboxTo(definition.DefType, definition.CharacterType, explicit, definition.charType);
             } else if (expected.clazz == int.class) {
-                return new Cast(definition.DefType, definition.IntegerType, explicit, null, definition.intType, null, null);
+                return Cast.unboxTo(definition.DefType, definition.IntegerType, explicit, definition.intType);
             } else if (expected.clazz == long.class) {
-                return new Cast(definition.DefType, definition.LongType, explicit, null, definition.longType, null, null);
+                return Cast.unboxTo(definition.DefType, definition.LongType, explicit, definition.longType);
             } else if (expected.clazz == float.class) {
-                return new Cast(definition.DefType, definition.FloatType, explicit, null, definition.floatType, null, null);
+                return Cast.unboxTo(definition.DefType, definition.FloatType, explicit, definition.floatType);
             } else if (expected.clazz == double.class) {
-                return new Cast(definition.DefType, definition.DoubleType, explicit, null, definition.doubleType, null, null);
+                return Cast.unboxTo(definition.DefType, definition.DoubleType, explicit, definition.doubleType);
             }
         } else if (actual.clazz == Object.class) {
             if (expected.clazz == byte.class && explicit && internal) {
-                return new Cast(definition.ObjectType, definition.ByteType, true, null, definition.byteType, null, null);
+                return Cast.unboxTo(definition.ObjectType, definition.ByteType, true, definition.byteType);
             } else if (expected.clazz == short.class && explicit && internal) {
-                return new Cast(definition.ObjectType, definition.ShortType, true, null, definition.shortType, null, null);
+                return Cast.unboxTo(definition.ObjectType, definition.ShortType, true, definition.shortType);
             } else if (expected.clazz == char.class && explicit && internal) {
-                return new Cast(definition.ObjectType, definition.CharacterType, true, null, definition.charType, null, null);
+                return Cast.unboxTo(definition.ObjectType, definition.CharacterType, true, definition.charType);
             } else if (expected.clazz == int.class && explicit && internal) {
-                return new Cast(definition.ObjectType, definition.IntegerType, true, null, definition.intType, null, null);
+                return Cast.unboxTo(definition.ObjectType, definition.IntegerType, true, definition.intType);
             } else if (expected.clazz == long.class && explicit && internal) {
-                return new Cast(definition.ObjectType, definition.LongType, true, null, definition.longType, null, null);
+                return Cast.unboxTo(definition.ObjectType, definition.LongType, true, definition.longType);
             } else if (expected.clazz == float.class && explicit && internal) {
-                return new Cast(definition.ObjectType, definition.FloatType, true, null, definition.floatType, null, null);
+                return Cast.unboxTo(definition.ObjectType, definition.FloatType, true, definition.floatType);
             } else if (expected.clazz == double.class && explicit && internal) {
-                return new Cast(definition.ObjectType, definition.DoubleType, true, null, definition.doubleType, null, null);
+                return Cast.unboxTo(definition.ObjectType, definition.DoubleType, true, definition.doubleType);
             }
         } else if (actual.clazz == Number.class) {
             if (expected.clazz == byte.class && explicit && internal) {
-                return new Cast(definition.NumberType, definition.ByteType, true, null, definition.byteType, null, null);
+                return Cast.unboxTo(definition.NumberType, definition.ByteType, true, definition.byteType);
             } else if (expected.clazz == short.class && explicit && internal) {
-                return new Cast(definition.NumberType, definition.ShortType, true, null, definition.shortType, null, null);
+                return Cast.unboxTo(definition.NumberType, definition.ShortType, true, definition.shortType);
             } else if (expected.clazz == char.class && explicit && internal) {
-                return new Cast(definition.NumberType, definition.CharacterType, true, null, definition.charType, null, null);
+                return Cast.unboxTo(definition.NumberType, definition.CharacterType, true, definition.charType);
             } else if (expected.clazz == int.class && explicit && internal) {
-                return new Cast(definition.NumberType, definition.IntegerType, true, null, definition.intType, null, null);
+                return Cast.unboxTo(definition.NumberType, definition.IntegerType, true, definition.intType);
             } else if (expected.clazz == long.class && explicit && internal) {
-                return new Cast(definition.NumberType, definition.LongType, true, null, definition.longType, null, null);
+                return Cast.unboxTo(definition.NumberType, definition.LongType, true, definition.longType);
             } else if (expected.clazz == float.class && explicit && internal) {
-                return new Cast(definition.NumberType, definition.FloatType, true, null, definition.floatType, null, null);
+                return Cast.unboxTo(definition.NumberType, definition.FloatType, true, definition.floatType);
             } else if (expected.clazz == double.class && explicit && internal) {
-                return new Cast(definition.NumberType, definition.DoubleType, true, null, definition.doubleType, null, null);
+                return Cast.unboxTo(definition.NumberType, definition.DoubleType, true, definition.doubleType);
             }
         } else if (actual.clazz == String.class) {
             if (expected.clazz == char.class && explicit) {
-                return new Cast(definition.StringType, definition.charType, true);
+                return Cast.standard(definition.StringType, definition.charType, true);
             }
         } else if (actual.clazz == boolean.class) {
             if (expected.dynamic) {
-                return new Cast(definition.BooleanType, definition.DefType, explicit, null, null, definition.booleanType, null);
+                return Cast.boxFrom(definition.BooleanType, definition.DefType, explicit, definition.booleanType);
             } else if (expected.clazz == Object.class && internal) {
-                return new Cast(definition.BooleanType, definition.ObjectType, explicit, null, null, definition.booleanType, null);
+                return Cast.boxFrom(definition.BooleanType, definition.ObjectType, explicit, definition.booleanType);
             } else if (expected.clazz == Boolean.class && internal) {
-                return new Cast(definition.booleanType, definition.booleanType, explicit, null, null, null, definition.booleanType);
+                return Cast.boxTo(definition.booleanType, definition.booleanType, explicit, definition.booleanType);
             }
         } else if (actual.clazz == byte.class) {
             if (expected.dynamic) {
-                return new Cast(definition.ByteType, definition.DefType, explicit, null, null, definition.byteType, null);
+                return Cast.boxFrom(definition.ByteType, definition.DefType, explicit, definition.byteType);
             } else if (expected.clazz == Object.class && internal) {
-                return new Cast(definition.ByteType, definition.ObjectType, explicit, null, null, definition.byteType, null);
+                return Cast.boxFrom(definition.ByteType, definition.ObjectType, explicit, definition.byteType);
             } else if (expected.clazz == Number.class && internal) {
-                return new Cast(definition.ByteType, definition.NumberType, explicit, null, null, definition.byteType, null);
+                return Cast.boxFrom(definition.ByteType, definition.NumberType, explicit, definition.byteType);
             } else if (expected.clazz == short.class) {
-                return new Cast(definition.byteType, definition.shortType, explicit);
+                return Cast.standard(definition.byteType, definition.shortType, explicit);
             } else if (expected.clazz == char.class && explicit) {
-                return new Cast(definition.byteType, definition.charType, true);
+                return Cast.standard(definition.byteType, definition.charType, true);
             } else if (expected.clazz == int.class) {
-                return new Cast(definition.byteType, definition.intType, explicit);
+                return Cast.standard(definition.byteType, definition.intType, explicit);
             } else if (expected.clazz == long.class) {
-                return new Cast(definition.byteType, definition.longType, explicit);
+                return Cast.standard(definition.byteType, definition.longType, explicit);
             } else if (expected.clazz == float.class) {
-                return new Cast(definition.byteType, definition.floatType, explicit);
+                return Cast.standard(definition.byteType, definition.floatType, explicit);
             } else if (expected.clazz == double.class) {
-                return new Cast(definition.byteType, definition.doubleType, explicit);
+                return Cast.standard(definition.byteType, definition.doubleType, explicit);
             } else if (expected.clazz == Byte.class && internal) {
-                return new Cast(definition.byteType, definition.byteType, explicit, null, null, null, definition.byteType);
+                return Cast.boxTo(definition.byteType, definition.byteType, explicit, definition.byteType);
             } else if (expected.clazz == Short.class && internal) {
-                return new Cast(definition.byteType, definition.shortType, explicit, null, null, null, definition.shortType);
+                return Cast.boxTo(definition.byteType, definition.shortType, explicit, definition.shortType);
             } else if (expected.clazz == Character.class && explicit && internal) {
-                return new Cast(definition.byteType, definition.charType, true, null, null, null, definition.charType);
+                return Cast.boxTo(definition.byteType, definition.charType, true, definition.charType);
             } else if (expected.clazz == Integer.class && internal) {
-                return new Cast(definition.byteType, definition.intType, explicit, null, null, null, definition.intType);
+                return Cast.boxTo(definition.byteType, definition.intType, explicit, definition.intType);
             } else if (expected.clazz == Long.class && internal) {
-                return new Cast(definition.byteType, definition.longType, explicit, null, null, null, definition.longType);
+                return Cast.boxTo(definition.byteType, definition.longType, explicit, definition.longType);
             } else if (expected.clazz == Float.class && internal) {
-                return new Cast(definition.byteType, definition.floatType, explicit, null, null, null, definition.floatType);
+                return Cast.boxTo(definition.byteType, definition.floatType, explicit, definition.floatType);
             } else if (expected.clazz == Double.class && internal) {
-                return new Cast(definition.byteType, definition.doubleType, explicit, null, null, null, definition.doubleType);
+                return Cast.boxTo(definition.byteType, definition.doubleType, explicit, definition.doubleType);
             }
         } else if (actual.clazz == short.class) {
             if (expected.dynamic) {
-                return new Cast(definition.ShortType, definition.DefType, explicit, null, null, definition.shortType, null);
+                return Cast.boxFrom(definition.ShortType, definition.DefType, explicit, definition.shortType);
             } else if (expected.clazz == Object.class && internal) {
-                return new Cast(definition.ShortType, definition.ObjectType, explicit, null, null, definition.shortType, null);
+                return Cast.boxFrom(definition.ShortType, definition.ObjectType, explicit, definition.shortType);
             } else if (expected.clazz == Number.class && internal) {
-                return new Cast(definition.ShortType, definition.NumberType, explicit, null, null, definition.shortType, null);
+                return Cast.boxFrom(definition.ShortType, definition.NumberType, explicit, definition.shortType);
             } else if (expected.clazz == byte.class && explicit) {
-                return new Cast(definition.shortType, definition.byteType, true);
+                return Cast.standard(definition.shortType, definition.byteType, true);
             } else if (expected.clazz == char.class && explicit) {
-                return new Cast(definition.shortType, definition.charType, true);
+                return Cast.standard(definition.shortType, definition.charType, true);
             } else if (expected.clazz == int.class) {
-                return new Cast(definition.shortType, definition.intType, explicit);
+                return Cast.standard(definition.shortType, definition.intType, explicit);
             } else if (expected.clazz == long.class) {
-                return new Cast(definition.shortType, definition.longType, explicit);
+                return Cast.standard(definition.shortType, definition.longType, explicit);
             } else if (expected.clazz == float.class) {
-                return new Cast(definition.shortType, definition.floatType, explicit);
+                return Cast.standard(definition.shortType, definition.floatType, explicit);
             } else if (expected.clazz == double.class) {
-                return new Cast(definition.shortType, definition.doubleType, explicit);
+                return Cast.standard(definition.shortType, definition.doubleType, explicit);
             } else if (expected.clazz == Byte.class && explicit && internal) {
-                return new Cast(definition.shortType, definition.byteType, true, null, null, null, definition.byteType);
+                return Cast.boxTo(definition.shortType, definition.byteType, true, definition.byteType);
             } else if (expected.clazz == Short.class && internal) {
-                return new Cast(definition.shortType, definition.shortType, explicit, null, null, null, definition.shortType);
+                return Cast.boxTo(definition.shortType, definition.shortType, explicit, definition.shortType);
             } else if (expected.clazz == Character.class && explicit && internal) {
-                return new Cast(definition.shortType, definition.charType, true, null, null, null, definition.charType);
+                return Cast.boxTo(definition.shortType, definition.charType, true, definition.charType);
             } else if (expected.clazz == Integer.class && internal) {
-                return new Cast(definition.shortType, definition.intType, explicit, null, null, null, definition.intType);
+                return Cast.boxTo(definition.shortType, definition.intType, explicit, definition.intType);
             } else if (expected.clazz == Long.class && internal) {
-                return new Cast(definition.shortType, definition.longType, explicit, null, null, null, definition.longType);
+                return Cast.boxTo(definition.shortType, definition.longType, explicit, definition.longType);
             } else if (expected.clazz == Float.class && internal) {
-                return new Cast(definition.shortType, definition.floatType, explicit, null, null, null, definition.floatType);
+                return Cast.boxTo(definition.shortType, definition.floatType, explicit, definition.floatType);
             } else if (expected.clazz == Double.class && internal) {
-                return new Cast(definition.shortType, definition.doubleType, explicit, null, null, null, definition.doubleType);
+                return Cast.boxTo(definition.shortType, definition.doubleType, explicit, definition.doubleType);
             }
         } else if (actual.clazz == char.class) {
             if (expected.dynamic) {
-                return new Cast(definition.CharacterType, definition.DefType, explicit, null, null, definition.charType, null);
+                return Cast.boxFrom(definition.CharacterType, definition.DefType, explicit, definition.charType);
             } else if (expected.clazz == Object.class && internal) {
-                return new Cast(definition.CharacterType, definition.ObjectType, explicit, null, null, definition.charType, null);
+                return Cast.boxFrom(definition.CharacterType, definition.ObjectType, explicit, definition.charType);
             } else if (expected.clazz == Number.class && internal) {
-                return new Cast(definition.CharacterType, definition.NumberType, explicit, null, null, definition.charType, null);
+                return Cast.boxFrom(definition.CharacterType, definition.NumberType, explicit, definition.charType);
             } else if (expected.clazz == String.class) {
-                return new Cast(definition.charType, definition.StringType, explicit);
+                return Cast.standard(definition.charType, definition.StringType, explicit);
             } else if (expected.clazz == byte.class && explicit) {
-                return new Cast(definition.charType, definition.byteType, true);
+                return Cast.standard(definition.charType, definition.byteType, true);
             } else if (expected.clazz == short.class && explicit) {
-                return new Cast(definition.charType, definition.shortType, true);
+                return Cast.standard(definition.charType, definition.shortType, true);
             } else if (expected.clazz == int.class) {
-                return new Cast(definition.charType, definition.intType, explicit);
+                return Cast.standard(definition.charType, definition.intType, explicit);
             } else if (expected.clazz == long.class) {
-                return new Cast(definition.charType, definition.longType, explicit);
+                return Cast.standard(definition.charType, definition.longType, explicit);
             } else if (expected.clazz == float.class) {
-                return new Cast(definition.charType, definition.floatType, explicit);
+                return Cast.standard(definition.charType, definition.floatType, explicit);
             } else if (expected.clazz == double.class) {
-                return new Cast(definition.charType, definition.doubleType, explicit);
+                return Cast.standard(definition.charType, definition.doubleType, explicit);
             } else if (expected.clazz == Byte.class && explicit && internal) {
-                return new Cast(definition.charType, definition.byteType, true, null, null, null, definition.byteType);
+                return Cast.boxTo(definition.charType, definition.byteType, true, definition.byteType);
             } else if (expected.clazz == Short.class && internal) {
-                return new Cast(definition.charType, definition.shortType, explicit, null, null, null, definition.shortType);
+                return Cast.boxTo(definition.charType, definition.shortType, explicit, definition.shortType);
             } else if (expected.clazz == Character.class && internal) {
-                return new Cast(definition.charType, definition.charType, true, null, null, null, definition.charType);
+                return Cast.boxTo(definition.charType, definition.charType, true, definition.charType);
             } else if (expected.clazz == Integer.class && internal) {
-                return new Cast(definition.charType, definition.intType, explicit, null, null, null, definition.intType);
+                return Cast.boxTo(definition.charType, definition.intType, explicit, definition.intType);
             } else if (expected.clazz == Long.class && internal) {
-                return new Cast(definition.charType, definition.longType, explicit, null, null, null, definition.longType);
+                return Cast.boxTo(definition.charType, definition.longType, explicit, definition.longType);
             } else if (expected.clazz == Float.class && internal) {
-                return new Cast(definition.charType, definition.floatType, explicit, null, null, null, definition.floatType);
+                return Cast.boxTo(definition.charType, definition.floatType, explicit, definition.floatType);
             } else if (expected.clazz == Double.class && internal) {
-                return new Cast(definition.charType, definition.doubleType, explicit, null, null, null, definition.doubleType);
+                return Cast.boxTo(definition.charType, definition.doubleType, explicit, definition.doubleType);
             }
         } else if (actual.clazz == int.class) {
             if (expected.dynamic) {
-                return new Cast(definition.IntegerType, definition.DefType, explicit, null, null, definition.intType, null);
+                return Cast.boxFrom(definition.IntegerType, definition.DefType, explicit, definition.intType);
             } else if (expected.clazz == Object.class && internal) {
-                return new Cast(definition.IntegerType, definition.ObjectType, explicit, null, null, definition.intType, null);
+                return Cast.boxFrom(definition.IntegerType, definition.ObjectType, explicit, definition.intType);
             } else if (expected.clazz == Number.class && internal) {
-                return new Cast(definition.IntegerType, definition.NumberType, explicit, null, null, definition.intType, null);
+                return Cast.boxFrom(definition.IntegerType, definition.NumberType, explicit, definition.intType);
             } else if (expected.clazz == byte.class && explicit) {
-                return new Cast(definition.intType, definition.byteType, true);
+                return Cast.standard(definition.intType, definition.byteType, true);
             } else if (expected.clazz == char.class && explicit) {
-                return new Cast(definition.intType, definition.charType, true);
+                return Cast.standard(definition.intType, definition.charType, true);
             } else if (expected.clazz == short.class && explicit) {
-                return new Cast(definition.intType, definition.shortType, true);
+                return Cast.standard(definition.intType, definition.shortType, true);
             } else if (expected.clazz == long.class) {
-                return new Cast(definition.intType, definition.longType, explicit);
+                return Cast.standard(definition.intType, definition.longType, explicit);
             } else if (expected.clazz == float.class) {
-                return new Cast(definition.intType, definition.floatType, explicit);
+                return Cast.standard(definition.intType, definition.floatType, explicit);
             } else if (expected.clazz == double.class) {
-                return new Cast(definition.intType, definition.doubleType, explicit);
+                return Cast.standard(definition.intType, definition.doubleType, explicit);
             } else if (expected.clazz == Byte.class && explicit && internal) {
-                return new Cast(definition.intType, definition.byteType, true, null, null, null, definition.byteType);
+                return Cast.boxTo(definition.intType, definition.byteType, true, definition.byteType);
             } else if (expected.clazz == Short.class && explicit && internal) {
-                return new Cast(definition.intType, definition.shortType, true, null, null, null, definition.shortType);
+                return Cast.boxTo(definition.intType, definition.shortType, true, definition.shortType);
             } else if (expected.clazz == Character.class && explicit && internal) {
-                return new Cast(definition.intType, definition.charType, true, null, null, null, definition.charType);
+                return Cast.boxTo(definition.intType, definition.charType, true, definition.charType);
             } else if (expected.clazz == Integer.class && internal) {
-                return new Cast(definition.intType, definition.intType, explicit, null, null, null, definition.intType);
+                return Cast.boxTo(definition.intType, definition.intType, explicit, definition.intType);
             } else if (expected.clazz == Long.class && internal) {
-                return new Cast(definition.intType, definition.longType, explicit, null, null, null, definition.longType);
+                return Cast.boxTo(definition.intType, definition.longType, explicit, definition.longType);
             } else if (expected.clazz == Float.class && internal) {
-                return new Cast(definition.intType, definition.floatType, explicit, null, null, null, definition.floatType);
+                return Cast.boxTo(definition.intType, definition.floatType, explicit, definition.floatType);
             } else if (expected.clazz == Double.class && internal) {
-                return new Cast(definition.intType, definition.doubleType, explicit, null, null, null, definition.doubleType);
+                return Cast.boxTo(definition.intType, definition.doubleType, explicit, definition.doubleType);
             }
         } else if (actual.clazz == long.class) {
             if (expected.dynamic) {
-                return new Cast(definition.LongType, definition.DefType, explicit, null, null, definition.longType, null);
+                return Cast.boxFrom(definition.LongType, definition.DefType, explicit, definition.longType);
             } else if (expected.clazz == Object.class && internal) {
-                return new Cast(definition.LongType, definition.ObjectType, explicit, null, null, definition.longType, null);
+                return Cast.boxFrom(definition.LongType, definition.ObjectType, explicit, definition.longType);
             } else if (expected.clazz == Number.class && internal) {
-                return new Cast(definition.LongType, definition.NumberType, explicit, null, null, definition.longType, null);
+                return Cast.boxFrom(definition.LongType, definition.NumberType, explicit, definition.longType);
             } else if (expected.clazz == byte.class && explicit) {
-                return new Cast(definition.longType, definition.byteType, true);
+                return Cast.standard(definition.longType, definition.byteType, true);
             } else if (expected.clazz == char.class && explicit) {
-                return new Cast(definition.longType, definition.charType, true);
+                return Cast.standard(definition.longType, definition.charType, true);
             } else if (expected.clazz == short.class && explicit) {
-                return new Cast(definition.longType, definition.shortType, true);
+                return Cast.standard(definition.longType, definition.shortType, true);
             } else if (expected.clazz == int.class && explicit) {
-                return new Cast(definition.longType, definition.intType, true);
+                return Cast.standard(definition.longType, definition.intType, true);
             } else if (expected.clazz == float.class) {
-                return new Cast(definition.longType, definition.floatType, explicit);
+                return Cast.standard(definition.longType, definition.floatType, explicit);
             } else if (expected.clazz == double.class) {
-                return new Cast(definition.longType, definition.doubleType, explicit);
+                return Cast.standard(definition.longType, definition.doubleType, explicit);
             } else if (expected.clazz == Byte.class && explicit && internal) {
-                return new Cast(definition.longType, definition.byteType, true, null, null, null, definition.byteType);
+                return Cast.boxTo(definition.longType, definition.byteType, true, definition.byteType);
             } else if (expected.clazz == Short.class && explicit && internal) {
-                return new Cast(definition.longType, definition.shortType, true, null, null, null, definition.shortType);
+                return Cast.boxTo(definition.longType, definition.shortType, true, definition.shortType);
             } else if (expected.clazz == Character.class && explicit && internal) {
-                return new Cast(definition.longType, definition.charType, true, null, null, null, definition.charType);
+                return Cast.boxTo(definition.longType, definition.charType, true, definition.charType);
             } else if (expected.clazz == Integer.class && explicit && internal) {
-                return new Cast(definition.longType, definition.intType, true, null, null, null, definition.intType);
+                return Cast.boxTo(definition.longType, definition.intType, true, definition.intType);
             } else if (expected.clazz == Long.class && internal) {
-                return new Cast(definition.longType, definition.longType, explicit, null, null, null, definition.longType);
+                return Cast.boxTo(definition.longType, definition.longType, explicit, definition.longType);
             } else if (expected.clazz == Float.class && internal) {
-                return new Cast(definition.longType, definition.floatType, explicit, null, null, null, definition.floatType);
+                return Cast.boxTo(definition.longType, definition.floatType, explicit, definition.floatType);
             } else if (expected.clazz == Double.class && internal) {
-                return new Cast(definition.longType, definition.doubleType, explicit, null, null, null, definition.doubleType);
+                return Cast.boxTo(definition.longType, definition.doubleType, explicit, definition.doubleType);
             }
         } else if (actual.clazz == float.class) {
             if (expected.dynamic) {
-                return new Cast(definition.FloatType, definition.DefType, explicit, null, null, definition.floatType, null);
+                return Cast.boxFrom(definition.FloatType, definition.DefType, explicit, definition.floatType);
             } else if (expected.clazz == Object.class && internal) {
-                return new Cast(definition.FloatType, definition.ObjectType, explicit, null, null, definition.floatType, null);
+                return Cast.boxFrom(definition.FloatType, definition.ObjectType, explicit, definition.floatType);
             } else if (expected.clazz == Number.class && internal) {
-                return new Cast(definition.FloatType, definition.NumberType, explicit, null, null, definition.floatType, null);
+                return Cast.boxFrom(definition.FloatType, definition.NumberType, explicit, definition.floatType);
             } else if (expected.clazz == byte.class && explicit) {
-                return new Cast(definition.floatType, definition.byteType, true);
+                return Cast.standard(definition.floatType, definition.byteType, true);
             } else if (expected.clazz == char.class && explicit) {
-                return new Cast(definition.floatType, definition.charType, true);
+                return Cast.standard(definition.floatType, definition.charType, true);
             } else if (expected.clazz == short.class && explicit) {
-                return new Cast(definition.floatType, definition.shortType, true);
+                return Cast.standard(definition.floatType, definition.shortType, true);
             } else if (expected.clazz == int.class && explicit) {
-                return new Cast(definition.floatType, definition.intType, true);
+                return Cast.standard(definition.floatType, definition.intType, true);
             } else if (expected.clazz == long.class && explicit) {
-                return new Cast(definition.floatType, definition.longType, true);
+                return Cast.standard(definition.floatType, definition.longType, true);
             } else if (expected.clazz == double.class) {
-                return new Cast(definition.floatType, definition.doubleType, explicit);
+                return Cast.standard(definition.floatType, definition.doubleType, explicit);
             } else if (expected.clazz == Byte.class && explicit && internal) {
-                return new Cast(definition.floatType, definition.byteType, true, null, null, null, definition.byteType);
+                return Cast.boxTo(definition.floatType, definition.byteType, true, definition.byteType);
             } else if (expected.clazz == Short.class && explicit && internal) {
-                return new Cast(definition.floatType, definition.shortType, true, null, null, null, definition.shortType);
+                return Cast.boxTo(definition.floatType, definition.shortType, true, definition.shortType);
             } else if (expected.clazz == Character.class && explicit && internal) {
-                return new Cast(definition.floatType, definition.charType, true, null, null, null, definition.charType);
+                return Cast.boxTo(definition.floatType, definition.charType, true, definition.charType);
             } else if (expected.clazz == Integer.class && explicit && internal) {
-                return new Cast(definition.floatType, definition.intType, true, null, null, null, definition.intType);
+                return Cast.boxTo(definition.floatType, definition.intType, true, definition.intType);
             } else if (expected.clazz == Long.class && explicit && internal) {
-                return new Cast(definition.floatType, definition.longType, true, null, null, null, definition.longType);
+                return Cast.boxTo(definition.floatType, definition.longType, true, definition.longType);
             } else if (expected.clazz == Float.class && internal) {
-                return new Cast(definition.floatType, definition.floatType, explicit, null, null, null, definition.floatType);
+                return Cast.boxTo(definition.floatType, definition.floatType, explicit, definition.floatType);
             } else if (expected.clazz == Double.class && internal) {
-                return new Cast(definition.floatType, definition.doubleType, explicit, null, null, null, definition.doubleType);
+                return Cast.boxTo(definition.floatType, definition.doubleType, explicit, definition.doubleType);
             }
         } else if (actual.clazz == double.class) {
             if (expected.dynamic) {
-                return new Cast(definition.DoubleType, definition.DefType, explicit, null, null, definition.doubleType, null);
+                return Cast.boxFrom(definition.DoubleType, definition.DefType, explicit, definition.doubleType);
             } else if (expected.clazz == Object.class && internal) {
-                return new Cast(definition.DoubleType, definition.ObjectType, explicit, null, null, definition.doubleType, null);
+                return Cast.boxFrom(definition.DoubleType, definition.ObjectType, explicit, definition.doubleType);
             } else if (expected.clazz == Number.class && internal) {
-                return new Cast(definition.DoubleType, definition.NumberType, explicit, null, null, definition.doubleType, null);
+                return Cast.boxFrom(definition.DoubleType, definition.NumberType, explicit, definition.doubleType);
             } else if (expected.clazz == byte.class && explicit) {
-                return new Cast(definition.doubleType, definition.byteType, true);
+                return Cast.standard(definition.doubleType, definition.byteType, true);
             } else if (expected.clazz == char.class && explicit) {
-                return new Cast(definition.doubleType, definition.charType, true);
+                return Cast.standard(definition.doubleType, definition.charType, true);
             } else if (expected.clazz == short.class && explicit) {
-                return new Cast(definition.doubleType, definition.shortType, true);
+                return Cast.standard(definition.doubleType, definition.shortType, true);
             } else if (expected.clazz == int.class && explicit) {
-                return new Cast(definition.doubleType, definition.intType, true);
+                return Cast.standard(definition.doubleType, definition.intType, true);
             } else if (expected.clazz == long.class && explicit) {
-                return new Cast(definition.doubleType, definition.longType, true);
+                return Cast.standard(definition.doubleType, definition.longType, true);
             } else if (expected.clazz == float.class && explicit) {
-                return new Cast(definition.doubleType, definition.floatType, true);
+                return Cast.standard(definition.doubleType, definition.floatType, true);
             } else if (expected.clazz == Byte.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.byteType, true, null, null, null, definition.byteType);
+                return Cast.boxTo(definition.doubleType, definition.byteType, true, definition.byteType);
             } else if (expected.clazz == Short.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.shortType, true, null, null, null, definition.shortType);
+                return Cast.boxTo(definition.doubleType, definition.shortType, true, definition.shortType);
             } else if (expected.clazz == Character.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.charType, true, null, null, null, definition.charType);
+                return Cast.boxTo(definition.doubleType, definition.charType, true, definition.charType);
             } else if (expected.clazz == Integer.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.intType, true, null, null, null, definition.intType);
+                return Cast.boxTo(definition.doubleType, definition.intType, true, definition.intType);
             } else if (expected.clazz == Long.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.longType, true, null, null, null, definition.longType);
+                return Cast.boxTo(definition.doubleType, definition.longType, true, definition.longType);
             } else if (expected.clazz == Float.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.floatType, true, null, null, null, definition.floatType);
+                return Cast.boxTo(definition.doubleType, definition.floatType, true, definition.floatType);
             } else if (expected.clazz == Double.class && internal) {
-                return new Cast(definition.doubleType, definition.doubleType, explicit, null, null, null, definition.doubleType);
+                return Cast.boxTo(definition.doubleType, definition.doubleType, explicit, definition.doubleType);
             }
         } else if (actual.clazz == Boolean.class) {
             if (expected.clazz == boolean.class && internal) {
-                return new Cast(definition.booleanType, definition.booleanType, explicit, definition.booleanType, null, null, null);
+                return Cast.unboxFrom(definition.booleanType, definition.booleanType, explicit, definition.booleanType);
             }
         } else if (actual.clazz == Byte.class) {
             if (expected.clazz == byte.class && internal) {
-                return new Cast(definition.byteType, definition.byteType, explicit, definition.byteType, null, null, null);
+                return Cast.unboxFrom(definition.byteType, definition.byteType, explicit, definition.byteType);
             } else if (expected.clazz == short.class && internal) {
-                return new Cast(definition.byteType, definition.shortType, explicit, definition.byteType, null, null, null);
+                return Cast.unboxFrom(definition.byteType, definition.shortType, explicit, definition.byteType);
             } else if (expected.clazz == char.class && explicit && internal) {
-                return new Cast(definition.byteType, definition.charType, true, definition.byteType, null, null, null);
+                return Cast.unboxFrom(definition.byteType, definition.charType, true, definition.byteType);
             } else if (expected.clazz == int.class && internal) {
-                return new Cast(definition.byteType, definition.intType, explicit, definition.byteType, null, null, null);
+                return Cast.unboxFrom(definition.byteType, definition.intType, explicit, definition.byteType);
             } else if (expected.clazz == long.class && internal) {
-                return new Cast(definition.byteType, definition.longType, explicit, definition.byteType, null, null, null);
+                return Cast.unboxFrom(definition.byteType, definition.longType, explicit, definition.byteType);
             } else if (expected.clazz == float.class && internal) {
-                return new Cast(definition.byteType, definition.floatType, explicit, definition.byteType, null, null, null);
+                return Cast.unboxFrom(definition.byteType, definition.floatType, explicit, definition.byteType);
             } else if (expected.clazz == double.class && internal) {
-                return new Cast(definition.byteType, definition.doubleType, explicit, definition.byteType, null, null, null);
+                return Cast.unboxFrom(definition.byteType, definition.doubleType, explicit, definition.byteType);
             }
         } else if (actual.clazz == Short.class) {
             if (expected.clazz == byte.class && explicit && internal) {
-                return new Cast(definition.shortType, definition.byteType, true, definition.shortType, null, null, null);
+                return Cast.unboxFrom(definition.shortType, definition.byteType, true, definition.shortType);
             } else if (expected.clazz == short.class && internal) {
-                return new Cast(definition.shortType, definition.shortType, explicit, definition.shortType, null, null, null);
+                return Cast.unboxFrom(definition.shortType, definition.shortType, explicit, definition.shortType);
             } else if (expected.clazz == char.class && explicit && internal) {
-                return new Cast(definition.shortType, definition.charType, true, definition.shortType, null, null, null);
+                return Cast.unboxFrom(definition.shortType, definition.charType, true, definition.shortType);
             } else if (expected.clazz == int.class && internal) {
-                return new Cast(definition.shortType, definition.intType, explicit, definition.shortType, null, null, null);
+                return Cast.unboxFrom(definition.shortType, definition.intType, explicit, definition.shortType);
             } else if (expected.clazz == long.class && internal) {
-                return new Cast(definition.shortType, definition.longType, explicit, definition.shortType, null, null, null);
+                return Cast.unboxFrom(definition.shortType, definition.longType, explicit, definition.shortType);
             } else if (expected.clazz == float.class && internal) {
-                return new Cast(definition.shortType, definition.floatType, explicit, definition.shortType, null, null, null);
+                return Cast.unboxFrom(definition.shortType, definition.floatType, explicit, definition.shortType);
             } else if (expected.clazz == double.class && internal) {
-                return new Cast(definition.shortType, definition.doubleType, explicit, definition.shortType, null, null, null);
+                return Cast.unboxFrom(definition.shortType, definition.doubleType, explicit, definition.shortType);
             }
         } else if (actual.clazz == Character.class) {
             if (expected.clazz == byte.class && explicit && internal) {
-                return new Cast(definition.charType, definition.byteType, true, definition.charType, null, null, null);
+                return Cast.unboxFrom(definition.charType, definition.byteType, true, definition.charType);
             } else if (expected.clazz == short.class && explicit && internal) {
-                return new Cast(definition.charType, definition.shortType, true, definition.charType, null, null, null);
+                return Cast.unboxFrom(definition.charType, definition.shortType, true, definition.charType);
             } else if (expected.clazz == char.class && internal) {
-                return new Cast(definition.charType, definition.charType, explicit, definition.charType, null, null, null);
+                return Cast.unboxFrom(definition.charType, definition.charType, explicit, definition.charType);
             } else if (expected.clazz == int.class && internal) {
-                return new Cast(definition.charType, definition.intType, explicit, definition.charType, null, null, null);
+                return Cast.unboxFrom(definition.charType, definition.intType, explicit, definition.charType);
             } else if (expected.clazz == long.class && internal) {
-                return new Cast(definition.charType, definition.longType, explicit, definition.charType, null, null, null);
+                return Cast.unboxFrom(definition.charType, definition.longType, explicit, definition.charType);
             } else if (expected.clazz == float.class && internal) {
-                return new Cast(definition.charType, definition.floatType, explicit, definition.charType, null, null, null);
+                return Cast.unboxFrom(definition.charType, definition.floatType, explicit, definition.charType);
             } else if (expected.clazz == double.class && internal) {
-                return new Cast(definition.charType, definition.doubleType, explicit, definition.charType, null, null, null);
+                return Cast.unboxFrom(definition.charType, definition.doubleType, explicit, definition.charType);
             }
         } else if (actual.clazz == Integer.class) {
             if (expected.clazz == byte.class && explicit && internal) {
-                return new Cast(definition.intType, definition.byteType, true, definition.intType, null, null, null);
+                return Cast.unboxFrom(definition.intType, definition.byteType, true, definition.intType);
             } else if (expected.clazz == short.class && explicit && internal) {
-                return new Cast(definition.intType, definition.shortType, true, definition.intType, null, null, null);
+                return Cast.unboxFrom(definition.intType, definition.shortType, true, definition.intType);
             } else if (expected.clazz == char.class && explicit && internal) {
-                return new Cast(definition.intType, definition.charType, true, definition.intType, null, null, null);
+                return Cast.unboxFrom(definition.intType, definition.charType, true, definition.intType);
             } else if (expected.clazz == int.class && internal) {
-                return new Cast(definition.intType, definition.intType, explicit, definition.intType, null, null, null);
+                return Cast.unboxFrom(definition.intType, definition.intType, explicit, definition.intType);
             } else if (expected.clazz == long.class && internal) {
-                return new Cast(definition.intType, definition.longType, explicit, definition.intType, null, null, null);
+                return Cast.unboxFrom(definition.intType, definition.longType, explicit, definition.intType);
             } else if (expected.clazz == float.class && internal) {
-                return new Cast(definition.intType, definition.floatType, explicit, definition.intType, null, null, null);
+                return Cast.unboxFrom(definition.intType, definition.floatType, explicit, definition.intType);
             } else if (expected.clazz == double.class && internal) {
-                return new Cast(definition.intType, definition.doubleType, explicit, definition.intType, null, null, null);
+                return Cast.unboxFrom(definition.intType, definition.doubleType, explicit, definition.intType);
             }
         } else if (actual.clazz == Long.class) {
             if (expected.clazz == byte.class && explicit && internal) {
-                return new Cast(definition.longType, definition.byteType, true, definition.longType, null, null, null);
+                return Cast.unboxFrom(definition.longType, definition.byteType, true, definition.longType);
             } else if (expected.clazz == short.class && explicit && internal) {
-                return new Cast(definition.longType, definition.shortType, true, definition.longType, null, null, null);
+                return Cast.unboxFrom(definition.longType, definition.shortType, true, definition.longType);
             } else if (expected.clazz == char.class && explicit && internal) {
-                return new Cast(definition.longType, definition.charType, true, definition.longType, null, null, null);
+                return Cast.unboxFrom(definition.longType, definition.charType, true, definition.longType);
             } else if (expected.clazz == int.class && explicit && internal) {
-                return new Cast(definition.longType, definition.intType, true, definition.longType, null, null, null);
+                return Cast.unboxFrom(definition.longType, definition.intType, true, definition.longType);
             } else if (expected.clazz == long.class && internal) {
-                return new Cast(definition.longType, definition.longType, explicit, definition.longType, null, null, null);
+                return Cast.unboxFrom(definition.longType, definition.longType, explicit, definition.longType);
             } else if (expected.clazz == float.class && internal) {
-                return new Cast(definition.longType, definition.floatType, explicit, definition.longType, null, null, null);
+                return Cast.unboxFrom(definition.longType, definition.floatType, explicit, definition.longType);
             } else if (expected.clazz == double.class && internal) {
-                return new Cast(definition.longType, definition.doubleType, explicit, definition.longType, null, null, null);
+                return Cast.unboxFrom(definition.longType, definition.doubleType, explicit, definition.longType);
             }
         } else if (actual.clazz == Float.class) {
             if (expected.clazz == byte.class && explicit && internal) {
-                return new Cast(definition.floatType, definition.byteType, true, definition.floatType, null, null, null);
+                return Cast.unboxFrom(definition.floatType, definition.byteType, true, definition.floatType);
             } else if (expected.clazz == short.class && explicit && internal) {
-                return new Cast(definition.floatType, definition.shortType, true, definition.floatType, null, null, null);
+                return Cast.unboxFrom(definition.floatType, definition.shortType, true, definition.floatType);
             } else if (expected.clazz == char.class && explicit && internal) {
-                return new Cast(definition.floatType, definition.charType, true, definition.floatType, null, null, null);
+                return Cast.unboxFrom(definition.floatType, definition.charType, true, definition.floatType);
             } else if (expected.clazz == int.class && explicit && internal) {
-                return new Cast(definition.floatType, definition.intType, true, definition.floatType, null, null, null);
+                return Cast.unboxFrom(definition.floatType, definition.intType, true, definition.floatType);
             } else if (expected.clazz == long.class && explicit && internal) {
-                return new Cast(definition.floatType, definition.longType, true, definition.floatType, null, null, null);
+                return Cast.unboxFrom(definition.floatType, definition.longType, true, definition.floatType);
             } else if (expected.clazz == float.class && internal) {
-                return new Cast(definition.floatType, definition.floatType, explicit, definition.floatType, null, null, null);
+                return Cast.unboxFrom(definition.floatType, definition.floatType, explicit, definition.floatType);
             } else if (expected.clazz == double.class && internal) {
-                return new Cast(definition.floatType, definition.doubleType, explicit, definition.floatType, null, null, null);
+                return Cast.unboxFrom(definition.floatType, definition.doubleType, explicit, definition.floatType);
             }
         } else if (actual.clazz == Double.class) {
             if (expected.clazz == byte.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.byteType, true, definition.doubleType, null, null, null);
+                return Cast.unboxFrom(definition.doubleType, definition.byteType, true, definition.doubleType);
             } else if (expected.clazz == short.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.shortType, true, definition.doubleType, null, null, null);
+                return Cast.unboxFrom(definition.doubleType, definition.shortType, true, definition.doubleType);
             } else if (expected.clazz == char.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.charType, true, definition.doubleType, null, null, null);
+                return Cast.unboxFrom(definition.doubleType, definition.charType, true, definition.doubleType);
             } else if (expected.clazz == int.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.intType, true, definition.doubleType, null, null, null);
+                return Cast.unboxFrom(definition.doubleType, definition.intType, true, definition.doubleType);
             } else if (expected.clazz == long.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.longType, true, definition.doubleType, null, null, null);
+                return Cast.unboxFrom(definition.doubleType, definition.longType, true, definition.doubleType);
             } else if (expected.clazz == float.class && explicit && internal) {
-                return new Cast(definition.doubleType, definition.floatType, true, definition.doubleType, null, null, null);
+                return Cast.unboxFrom(definition.doubleType, definition.floatType, true, definition.doubleType);
             } else if (expected.clazz == double.class && internal) {
-                return new Cast(definition.doubleType, definition.doubleType, explicit, definition.doubleType, null, null, null);
+                return Cast.unboxFrom(definition.doubleType, definition.doubleType, explicit, definition.doubleType);
             }
         }
 
@@ -468,7 +468,7 @@ public final class AnalyzerCaster {
                 (actual.clazz != void.class && expected.dynamic) ||
                 expected.clazz.isAssignableFrom(actual.clazz)    ||
                 (actual.clazz.isAssignableFrom(expected.clazz) && explicit)) {
-            return new Cast(actual, expected, explicit);
+            return Cast.standard(actual, expected, explicit);
         } else {
             throw location.createError(new ClassCastException("Cannot cast from [" + actual.name + "] to [" + expected.name + "]."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 43de306a7a8..853c836f9cf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -384,6 +384,31 @@ public final class Definition {
     }
 
     public static class Cast {
+        /** Create a standard cast with no boxing/unboxing. */
+        public static Cast standard(Type from, Type to, boolean explicit) {
+            return new Cast(from, to, explicit, null, null, null, null);
+        }
+
+        /** Create a cast where the from type will be unboxed, and then the cast will be performed. */
+        public static Cast unboxFrom(Type from, Type to, boolean explicit, Type unboxFrom) {
+            return new Cast(from, to, explicit, unboxFrom, null, null, null);
+        }
+
+        /** Create a cast where the to type will be unboxed, and then the cast will be performed. */
+        public static Cast unboxTo(Type from, Type to, boolean explicit, Type unboxTo) {
+            return new Cast(from, to, explicit, null, unboxTo, null, null);
+        }
+
+        /** Create a cast where the from type will be boxed, and then the cast will be performed. */
+        public static Cast boxFrom(Type from, Type to, boolean explicit, Type boxFrom) {
+            return new Cast(from, to, explicit, null, null, boxFrom, null);
+        }
+
+        /** Create a cast where the to type will be boxed, and then the cast will be performed. */
+        public static Cast boxTo(Type from, Type to, boolean explicit, Type boxTo) {
+            return new Cast(from, to, explicit, null, null, null, boxTo);
+        }
+
         public final Type from;
         public final Type to;
         public final boolean explicit;
@@ -392,18 +417,7 @@ public final class Definition {
         public final Type boxFrom;
         public final Type boxTo;
 
-        public Cast(final Type from, final Type to, final boolean explicit) {
-            this.from = from;
-            this.to = to;
-            this.explicit = explicit;
-            this.unboxFrom = null;
-            this.unboxTo = null;
-            this.boxFrom = null;
-            this.boxTo = null;
-        }
-
-        public Cast(final Type from, final Type to, final boolean explicit,
-                    final Type unboxFrom, final Type unboxTo, final Type boxFrom, final Type boxTo) {
+        private Cast(Type from, Type to, boolean explicit, Type unboxFrom, Type unboxTo, Type boxFrom, Type boxTo) {
             this.from = from;
             this.to = to;
             this.explicit = explicit;
@@ -412,7 +426,6 @@ public final class Definition {
             this.boxFrom = boxFrom;
             this.boxTo = boxTo;
         }
-
     }
 
     public static final class RuntimeClass {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
index a18c27cdef5..2bd9753ea26 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java
@@ -161,12 +161,12 @@ public class NodeToStringTests extends ESTestCase {
     public void testECast() {
         Location l = new Location(getTestName(), 0);
         AExpression child = new EConstant(l, "test");
-        Cast cast = new Cast(Definition.DEFINITION.StringType, Definition.DEFINITION.IntegerType, true);
+        Cast cast = Cast.standard(Definition.DEFINITION.StringType, Definition.DEFINITION.IntegerType, true);
         assertEquals("(ECast java.lang.Integer (EConstant String 'test'))", new ECast(l, child, cast).toString());
 
         l = new Location(getTestName(), 1);
         child = new EBinary(l, Operation.ADD, new EConstant(l, "test"), new EConstant(l, 12));
-        cast = new Cast(Definition.DEFINITION.IntegerType, Definition.DEFINITION.BooleanType, true);
+        cast = Cast.standard(Definition.DEFINITION.IntegerType, Definition.DEFINITION.BooleanType, true);
         assertEquals("(ECast java.lang.Boolean (EBinary (EConstant String 'test') + (EConstant Integer 12)))",
             new ECast(l, child, cast).toString());
     }

From c541a0c60e1f1bbafb773094b67025dc7e86bbb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= 
Date: Thu, 14 Dec 2017 22:18:37 +0100
Subject: [PATCH 268/297] Add skip versions for rank_eval yaml tests

---
 .../resources/rest-api-spec/test/rank_eval/10_basic.yml  | 9 +++++++++
 .../resources/rest-api-spec/test/rank_eval/20_dcg.yml    | 4 ++++
 .../rest-api-spec/test/rank_eval/30_failures.yml         | 4 ++++
 .../rest-api-spec/test/rank-eval/30_template.yml         | 5 +++++
 4 files changed, 22 insertions(+)

diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
index 1c5b82468cc..dc91f63420c 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
@@ -1,5 +1,10 @@
 ---
 "Response format":
+
+  - skip:
+      version: " - 6.99.99"
+      reason: the ranking evaluation feature is only available on 7.0
+
   - do:
      indices.create:
          index: foo
@@ -84,6 +89,10 @@
 ---
 "Mean Reciprocal Rank":
 
+  - skip:
+      version: " - 6.99.99"
+      reason: the ranking evaluation feature is only available on 7.0
+
   - do:
      indices.create:
          index: foo
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml
index c6c1c87e753..37c0b5897f5 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml
@@ -1,6 +1,10 @@
 ---
 "Response format":
 
+  - skip:
+      version: " - 6.99.99"
+      reason: the ranking evaluation feature is only available on 7.0
+
   - do:
       index:
         index:   foo
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml
index 130da28f3b1..55efcdd104a 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml
@@ -1,6 +1,10 @@
 ---
 "Response format":
 
+  - skip:
+      version: " - 6.99.99"
+      reason: the ranking evaluation feature is only available on 7.0
+
   - do:
       index:
         index:   foo
diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yml b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yml
index f6e77115a23..0a59b7d0733 100644
--- a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yml
+++ b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yml
@@ -1,5 +1,10 @@
 ---
 "Template request":
+
+  - skip:
+      version: " - 6.99.99"
+      reason: the ranking evaluation feature is only available on 7.0
+
   - do:
      indices.create:
          index: foo

From f33f9612a7c09e464a01eb9c0cf50ced909c67fe Mon Sep 17 00:00:00 2001
From: Tim Brooks 
Date: Thu, 14 Dec 2017 14:37:41 -0700
Subject: [PATCH 269/297] Remove potential nio selector leak (#27825)

When an ESSelector is created an underlying nio selector is opened. This
selector is closed by the event loop after close has been signalled by
another thread.

However, there is a possibility that an ESSelector is created and some
exception in the startup process prevents it from ever being started
(however, close will still be called). The allows the selector to leak.

This commit addresses this issue by having the signalling thread close
the selector if the event loop is not running when close is signalled.
---
 .../java/org/elasticsearch/transport/nio/ESSelector.java    | 5 ++++-
 .../org/elasticsearch/transport/nio/ESSelectorTests.java    | 6 ++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/ESSelector.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/ESSelector.java
index 667107f7b3e..91e308a33b5 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/ESSelector.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/ESSelector.java
@@ -177,8 +177,11 @@ public abstract class ESSelector implements Closeable {
                 try {
                     exitedLoop.await();
                 } catch (InterruptedException e) {
-                    eventHandler.uncaughtException(e);
+                    Thread.currentThread().interrupt();
+                    throw new IllegalStateException("Thread was interrupted while waiting for selector to close", e);
                 }
+            } else if (selector.isOpen()) {
+                selector.close();
             }
         }
     }
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/ESSelectorTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/ESSelectorTests.java
index 69c2c00489d..6459447c1a8 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/ESSelectorTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/ESSelectorTests.java
@@ -81,6 +81,12 @@ public class ESSelectorTests extends ESTestCase {
         verify(handler).selectException(ioException);
     }
 
+    public void testSelectorClosedIfOpenAndEventLoopNotRunning() throws IOException {
+        when(rawSelector.isOpen()).thenReturn(true);
+        selector.close();
+        verify(rawSelector).close();
+    }
+
     private static class TestSelector extends ESSelector {
 
         TestSelector(EventHandler eventHandler, Selector selector) throws IOException {

From af8bd8bbcf9942aa68fadccc78510606448e13f5 Mon Sep 17 00:00:00 2001
From: Alex Crome 
Date: Fri, 15 Dec 2017 01:41:39 +0000
Subject: [PATCH 270/297] Fix custom service names when installing on windows

We document that users can set custom service names on Windows. Alas,
the functionality does not work. This commit fixes the issue by passing
the environment variable SERVICE_ID as the service name otherwise
defaulting to elasticsearch-service-x64.

Relates #25255
---
 distribution/src/main/resources/bin/elasticsearch-service.bat | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch-service.bat b/distribution/src/main/resources/bin/elasticsearch-service.bat
index 6218d120627..72b5f988794 100644
--- a/distribution/src/main/resources/bin/elasticsearch-service.bat
+++ b/distribution/src/main/resources/bin/elasticsearch-service.bat
@@ -6,7 +6,7 @@ setlocal enableextensions
 call "%~dp0elasticsearch-env.bat" || exit /b 1
 
 set EXECUTABLE=%ES_HOME%\bin\elasticsearch-service-x64.exe
-set SERVICE_ID=elasticsearch-service-x64
+if "%SERVICE_ID%" == "" set SERVICE_ID=elasticsearch-service-x64
 set ARCH=64-bit
 
 if EXIST "%EXECUTABLE%" goto okExe

From e9160fc014f3c4558629ec613b53e34d6983574e Mon Sep 17 00:00:00 2001
From: Martijn van Groningen 
Date: Wed, 29 Nov 2017 15:17:03 +0100
Subject: [PATCH 271/297] percolator: also extract match_all queries

I've seen several cases where match_all queries were being used inside percolator queries,
because these queries were created generated by other systems.

Extracting these queries will allow the percolator at query time in a filter context
to skip over these queries without parsing or validating that these queries actually
match with the document being percolated.
---
 .../percolator/PercolatorFieldMapper.java     | 27 +++++---
 .../percolator/QueryAnalyzer.java             | 35 +++++++++-
 .../percolator/CandidateQueryTests.java       | 64 ++++++++++++++++++
 .../percolator/QueryAnalyzerTests.java        | 65 ++++++++++++++++---
 4 files changed, 170 insertions(+), 21 deletions(-)

diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index 1df6935c61a..e44a36cd267 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -458,20 +458,27 @@ public class PercolatorFieldMapper extends FieldMapper {
             doc.add(new Field(pft.extractionResultField.name(), EXTRACTION_FAILED, extractionResultField.fieldType()));
             return;
         }
-        for (QueryAnalyzer.QueryExtraction term : result.extractions) {
-            if (term.term != null) {
+        for (QueryAnalyzer.QueryExtraction extraction : result.extractions) {
+            if (extraction.term != null) {
                 BytesRefBuilder builder = new BytesRefBuilder();
-                builder.append(new BytesRef(term.field()));
+                builder.append(new BytesRef(extraction.field()));
                 builder.append(FIELD_VALUE_SEPARATOR);
-                builder.append(term.bytes());
+                builder.append(extraction.bytes());
                 doc.add(new Field(queryTermsField.name(), builder.toBytesRef(), queryTermsField.fieldType()));
-            } else if (term.range != null) {
-                byte[] min = term.range.lowerPoint;
-                byte[] max = term.range.upperPoint;
-                doc.add(new BinaryRange(rangeFieldMapper.name(), encodeRange(term.range.fieldName, min, max)));
+            } else if (extraction.range != null) {
+                byte[] min = extraction.range.lowerPoint;
+                byte[] max = extraction.range.upperPoint;
+                doc.add(new BinaryRange(rangeFieldMapper.name(), encodeRange(extraction.range.fieldName, min, max)));
             }
         }
-        if (result.verified) {
+
+        Version indexVersionCreated = context.mapperService().getIndexSettings().getIndexVersionCreated();
+        if (result.matchAllDocs) {
+            doc.add(new Field(extractionResultField.name(), EXTRACTION_FAILED, extractionResultField.fieldType()));
+            if (result.verified) {
+                doc.add(new Field(extractionResultField.name(), EXTRACTION_COMPLETE, extractionResultField.fieldType()));
+            }
+        } else if (result.verified) {
             doc.add(new Field(extractionResultField.name(), EXTRACTION_COMPLETE, extractionResultField.fieldType()));
         } else {
             doc.add(new Field(extractionResultField.name(), EXTRACTION_PARTIAL, extractionResultField.fieldType()));
@@ -481,7 +488,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         for (IndexableField field : fields) {
             context.doc().add(field);
         }
-        if (context.mapperService().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0)) {
+        if (indexVersionCreated.onOrAfter(Version.V_6_1_0)) {
             doc.add(new NumericDocValuesField(minimumShouldMatchFieldMapper.name(), result.minimumShouldMatch));
         }
     }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
index 284eca1b59a..ae619a1d494 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
@@ -29,6 +29,7 @@ import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
@@ -70,6 +71,7 @@ final class QueryAnalyzer {
     static {
         Map, BiFunction> map = new HashMap<>();
         map.put(MatchNoDocsQuery.class, matchNoDocsQuery());
+        map.put(MatchAllDocsQuery.class, matchAllDocsQuery());
         map.put(ConstantScoreQuery.class, constantScoreQuery());
         map.put(BoostQuery.class, boostQuery());
         map.put(TermQuery.class, termQuery());
@@ -142,6 +144,10 @@ final class QueryAnalyzer {
         return (query, version) -> new Result(true, Collections.emptySet(), 1);
     }
 
+    private static BiFunction matchAllDocsQuery() {
+        return (query, version) -> new Result(true, true);
+    }
+
     private static BiFunction constantScoreQuery() {
         return (query, boosts) -> {
             Query wrappedQuery = ((ConstantScoreQuery) query).getQuery();
@@ -356,6 +362,7 @@ final class QueryAnalyzer {
                         int msm = 0;
                         boolean requiredShouldClauses = minimumShouldMatch > 0 && numOptionalClauses > 0;
                         boolean verified = uqe == null && numProhibitedClauses == 0 && requiredShouldClauses == false;
+                        boolean matchAllDocs = true;
                         Set extractions = new HashSet<>();
                         Set seenRangeFields = new HashSet<>();
                         for (Result result : results) {
@@ -376,9 +383,14 @@ final class QueryAnalyzer {
                                 msm += result.minimumShouldMatch;
                             }
                             verified &= result.verified;
+                            matchAllDocs &= result.matchAllDocs;
                             extractions.addAll(result.extractions);
                         }
-                        return new Result(verified, extractions, msm);
+                        if (matchAllDocs) {
+                            return new Result(matchAllDocs, verified);
+                        } else {
+                            return new Result(verified, extractions, msm);
+                        }
                     }
                 } else {
                     Set bestClause = null;
@@ -498,12 +510,15 @@ final class QueryAnalyzer {
         if (version.before(Version.V_6_1_0)) {
             verified &= requiredShouldClauses <= 1;
         }
-
+        int numMatchAllClauses = 0;
         Set terms = new HashSet<>();
         for (int i = 0; i < disjunctions.size(); i++) {
             Query disjunct = disjunctions.get(i);
             Result subResult = analyze(disjunct, version);
             verified &= subResult.verified;
+            if (subResult.matchAllDocs) {
+                numMatchAllClauses++;
+            }
             terms.addAll(subResult.extractions);
 
             QueryExtraction[] t = subResult.extractions.toArray(new QueryExtraction[1]);
@@ -512,6 +527,7 @@ final class QueryAnalyzer {
                 rangeFieldNames[i] = t[0].range.fieldName;
             }
         }
+        boolean matchAllDocs = numMatchAllClauses > 0 && numMatchAllClauses >= requiredShouldClauses;
 
         int msm = 0;
         if (version.onOrAfter(Version.V_6_1_0)) {
@@ -532,7 +548,11 @@ final class QueryAnalyzer {
         } else {
             msm = 1;
         }
-        return new Result(verified, terms, msm);
+        if (matchAllDocs) {
+            return new Result(matchAllDocs, verified);
+        } else {
+            return new Result(verified, terms, msm);
+        }
     }
 
     static Set selectBestExtraction(Set extractions1, Set extractions2) {
@@ -619,11 +639,20 @@ final class QueryAnalyzer {
         final Set extractions;
         final boolean verified;
         final int minimumShouldMatch;
+        final boolean matchAllDocs;
 
         Result(boolean verified, Set extractions, int minimumShouldMatch) {
             this.extractions = extractions;
             this.verified = verified;
             this.minimumShouldMatch = minimumShouldMatch;
+            this.matchAllDocs = false;
+        }
+
+        Result(boolean matchAllDocs, boolean verified) {
+            this.extractions = Collections.emptySet();
+            this.verified = verified;
+            this.minimumShouldMatch = 0;
+            this.matchAllDocs = matchAllDocs;
         }
 
     }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
index 4ecd82fd876..fea5bd893a9 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
@@ -56,6 +56,8 @@ import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
@@ -193,6 +195,8 @@ public class CandidateQueryTests extends ESSingleNodeTestCase {
             builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
             if (randomBoolean()) {
                 builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
+            } else if (randomBoolean()) {
+                builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST_NOT);
             }
             return builder.build();
         });
@@ -202,6 +206,20 @@ public class CandidateQueryTests extends ESSingleNodeTestCase {
             builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
             if (randomBoolean()) {
                 builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
+            } else if (randomBoolean()) {
+                builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST_NOT);
+            }
+            return builder.build();
+        });
+        queryFunctions.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+            builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
+            if (randomBoolean()) {
+                builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+            }
+            if (randomBoolean()) {
+                builder.setMinimumNumberShouldMatch(2);
             }
             return builder.build();
         });
@@ -467,6 +485,52 @@ public class CandidateQueryTests extends ESSingleNodeTestCase {
         duelRun(queryStore, memoryIndex, shardSearcher);
     }
 
+    public void testPercolateMatchAll() throws Exception {
+        List docs = new ArrayList<>();
+        addQuery(new MatchAllDocsQuery(), docs);
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        builder.add(new TermQuery(new Term("field", "value1")), BooleanClause.Occur.MUST);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+        addQuery(builder.build(), docs);
+        builder = new BooleanQuery.Builder();
+        builder.add(new TermQuery(new Term("field", "value2")), BooleanClause.Occur.MUST);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+        addQuery(builder.build(), docs);
+        builder = new BooleanQuery.Builder();
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST_NOT);
+        addQuery(builder.build(), docs);
+        builder = new BooleanQuery.Builder();
+        builder.add(new TermQuery(new Term("field", "value2")), BooleanClause.Occur.SHOULD);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+        addQuery(builder.build(), docs);
+        indexWriter.addDocuments(docs);
+        indexWriter.close();
+        directoryReader = DirectoryReader.open(directory);
+        IndexSearcher shardSearcher = newSearcher(directoryReader);
+        shardSearcher.setQueryCache(null);
+
+        MemoryIndex memoryIndex = new MemoryIndex();
+        memoryIndex.addField("field", "value1", new WhitespaceAnalyzer());
+        IndexSearcher percolateSearcher = memoryIndex.createSearcher();
+        PercolateQuery query = (PercolateQuery) fieldType.percolateQuery("_name", queryStore,
+            Collections.singletonList(new BytesArray("{}")), percolateSearcher, Version.CURRENT);
+        TopDocs topDocs = shardSearcher.search(query, 10, new Sort(SortField.FIELD_DOC), true, true);
+        assertEquals(3L, topDocs.totalHits);
+        assertEquals(3, topDocs.scoreDocs.length);
+        assertEquals(0, topDocs.scoreDocs[0].doc);
+        assertEquals(1, topDocs.scoreDocs[1].doc);
+        assertEquals(4, topDocs.scoreDocs[2].doc);
+
+        topDocs = shardSearcher.search(new ConstantScoreQuery(query), 10);
+        assertEquals(3L, topDocs.totalHits);
+        assertEquals(3, topDocs.scoreDocs.length);
+        assertEquals(0, topDocs.scoreDocs[0].doc);
+        assertEquals(1, topDocs.scoreDocs[1].doc);
+        assertEquals(4, topDocs.scoreDocs[2].doc);
+    }
+
     public void testPercolateSmallAndLargeDocument() throws Exception {
         List docs = new ArrayList<>();
         BooleanQuery.Builder builder = new BooleanQuery.Builder();
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
index f2f5a4e5861..34637465af5 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
@@ -594,13 +594,18 @@ public class QueryAnalyzerTests extends ESTestCase {
     }
 
     public void testExtractQueryMetadata_matchAllDocsQuery() {
-        expectThrows(UnsupportedQueryException.class, () -> analyze(new MatchAllDocsQuery(), Version.CURRENT));
+        Result result = analyze(new MatchAllDocsQuery(), Version.CURRENT);
+        assertThat(result.verified, is(true));
+        assertThat(result.matchAllDocs, is(true));
+        assertThat(result.minimumShouldMatch, equalTo(0));
+        assertThat(result.extractions.size(), equalTo(0));
 
         BooleanQuery.Builder builder = new BooleanQuery.Builder();
         builder.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.MUST);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
-        Result result = analyze(builder.build(), Version.CURRENT);
-        assertThat(result.verified, is(false));
+        result = analyze(builder.build(), Version.CURRENT);
+        assertThat(result.verified, is(true));
+        assertThat(result.matchAllDocs, is(false));
         assertThat(result.minimumShouldMatch, equalTo(1));
         assertTermsEqual(result.extractions, new Term("field", "value"));
 
@@ -609,34 +614,78 @@ public class QueryAnalyzerTests extends ESTestCase {
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
         BooleanQuery bq1 = builder.build();
-        expectThrows(UnsupportedQueryException.class, () -> analyze(bq1, Version.CURRENT));
+        result = analyze(bq1, Version.CURRENT);
+        assertThat(result.verified, is(true));
+        assertThat(result.matchAllDocs, is(true));
+        assertThat(result.minimumShouldMatch, equalTo(0));
+        assertThat(result.extractions.size(), equalTo(0));
 
         builder = new BooleanQuery.Builder();
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST_NOT);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
         BooleanQuery bq2 = builder.build();
-        expectThrows(UnsupportedQueryException.class, () -> analyze(bq2, Version.CURRENT));
+        result = analyze(bq2, Version.CURRENT);
+        assertThat(result.verified, is(false));
+        assertThat(result.matchAllDocs, is(true));
+        assertThat(result.minimumShouldMatch, equalTo(0));
+        assertThat(result.extractions.size(), equalTo(0));
 
         builder = new BooleanQuery.Builder();
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         BooleanQuery bq3 = builder.build();
-        expectThrows(UnsupportedQueryException.class, () -> analyze(bq3, Version.CURRENT));
+        result = analyze(bq3, Version.CURRENT);
+        assertThat(result.verified, is(true));
+        assertThat(result.matchAllDocs, is(true));
+        assertThat(result.minimumShouldMatch, equalTo(0));
+        assertThat(result.extractions.size(), equalTo(0));
 
         builder = new BooleanQuery.Builder();
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST_NOT);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         BooleanQuery bq4 = builder.build();
-        expectThrows(UnsupportedQueryException.class, () -> analyze(bq4, Version.CURRENT));
+        result = analyze(bq4, Version.CURRENT);
+        assertThat(result.verified, is(false));
+        assertThat(result.matchAllDocs, is(true));
+        assertThat(result.minimumShouldMatch, equalTo(0));
+        assertThat(result.extractions.size(), equalTo(0));
 
         builder = new BooleanQuery.Builder();
         builder.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.SHOULD);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         BooleanQuery bq5 = builder.build();
-        expectThrows(UnsupportedQueryException.class, () -> analyze(bq5, Version.CURRENT));
+        result = analyze(bq5, Version.CURRENT);
+        assertThat(result.verified, is(true));
+        assertThat(result.matchAllDocs, is(true));
+        assertThat(result.minimumShouldMatch, equalTo(0));
+        assertThat(result.extractions.size(), equalTo(0));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+        builder.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.SHOULD);
+        builder.setMinimumNumberShouldMatch(2);
+        BooleanQuery bq6 = builder.build();
+        result = analyze(bq6, Version.CURRENT);
+        assertThat(result.verified, is(true));
+        assertThat(result.matchAllDocs, is(false));
+        assertThat(result.minimumShouldMatch, equalTo(1));
+        assertThat(result.extractions.size(), equalTo(1));
+        assertTermsEqual(result.extractions, new Term("field", "value"));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+        builder.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.SHOULD);
+        builder.setMinimumNumberShouldMatch(2);
+        BooleanQuery bq7 = builder.build();
+        result = analyze(bq7, Version.CURRENT);
+        assertThat(result.verified, is(true));
+        assertThat(result.matchAllDocs, is(true));
+        assertThat(result.minimumShouldMatch, equalTo(0));
+        assertThat(result.extractions.size(), equalTo(0));
     }
 
     public void testExtractQueryMetadata_unsupportedQuery() {

From 54b1fed5b381610bd4899067d66e199527c80e45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= 
Date: Fri, 15 Dec 2017 08:56:59 +0100
Subject: [PATCH 272/297] Corrected ByteSizeValue bwc serialization version
 after backport to 6.x

---
 .../java/org/elasticsearch/common/unit/ByteSizeValue.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
index 9fb7a3852d7..0981d0c4d72 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java
@@ -39,7 +39,7 @@ public class ByteSizeValue implements Writeable, Comparable {
     private final ByteSizeUnit unit;
 
     public ByteSizeValue(StreamInput in) throws IOException {
-        if (in.getVersion().before(Version.V_7_0_0_alpha1)) {
+        if (in.getVersion().before(Version.V_6_2_0)) {
             size = in.readVLong();
             unit = ByteSizeUnit.BYTES;
         } else {
@@ -50,7 +50,7 @@ public class ByteSizeValue implements Writeable, Comparable {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().before(Version.V_7_0_0_alpha1)) {
+        if (out.getVersion().before(Version.V_6_2_0)) {
             out.writeVLong(getBytes());
         } else {
             out.writeZLong(size);

From 1e5d3787e5e3a12d7392384bf84c8a0ddb808afd Mon Sep 17 00:00:00 2001
From: Simon Willnauer 
Date: Fri, 15 Dec 2017 09:25:19 +0100
Subject: [PATCH 273/297] [TEST] Don't start thread before checking for pending
 refresh

If we start the thread too early it registers a refresh listener and
that causes out assertion to fail if there is a zero timeout.
Closes #27769
---
 .../index/shard/IndexShardIT.java             | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
index ce210765c5c..e02b6c04a89 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
@@ -614,16 +614,6 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         assertFalse(indexService.getIndexSettings().isExplicitRefresh());
         ensureGreen();
         AtomicInteger totalNumDocs = new AtomicInteger(Integer.MAX_VALUE);
-        CountDownLatch started = new CountDownLatch(1);
-        Thread t = new Thread(() -> {
-            SearchResponse searchResponse;
-            started.countDown();
-            do {
-               searchResponse = client().prepareSearch().get();
-           } while (searchResponse.getHits().totalHits != totalNumDocs.get());
-        });
-        t.start();
-        started.await();
         assertNoSearchHits(client().prepareSearch().get());
         int numDocs = scaledRandomIntBetween(25, 100);
         totalNumDocs.set(numDocs);
@@ -641,6 +631,16 @@ public class IndexShardIT extends ESSingleNodeTestCase {
             // we can't assert on hasRefreshed since it might have been refreshed in the background on the shard concurrently
             assertFalse(shard.isSearchIdle());
         }
+        CountDownLatch started = new CountDownLatch(1);
+        Thread t = new Thread(() -> {
+            SearchResponse searchResponse;
+            started.countDown();
+            do {
+                searchResponse = client().prepareSearch().get();
+            } while (searchResponse.getHits().totalHits != totalNumDocs.get());
+        });
+        t.start();
+        started.await();
         assertHitCount(client().prepareSearch().get(), 1);
         for (int i = 1; i < numDocs; i++) {
             client().prepareIndex("test", "test", "" + i).setSource("{\"foo\" : \"bar\"}", XContentType.JSON)

From 5f74e81f2975e4f305f2ed2082a1fd14a9b4ef75 Mon Sep 17 00:00:00 2001
From: Alex Benusovich 
Date: Fri, 15 Dec 2017 00:42:36 -0800
Subject: [PATCH 274/297] [Docs] Clarify version for restoring old indices.
 (#27800)

Sentence should say that an index created in 2.x cannot be restored in a 6.x cluster even if 5.x cluster created the snapshot.
---
 docs/reference/upgrade.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/upgrade.asciidoc b/docs/reference/upgrade.asciidoc
index 5218856d0c9..4b03364657b 100644
--- a/docs/reference/upgrade.asciidoc
+++ b/docs/reference/upgrade.asciidoc
@@ -58,7 +58,7 @@ Elasticsearch 2.x, but not those created in 1.x or before.
 
 This also applies to indices backed up with <>. If an index was originally created in 2.x, it cannot be
-restored to a 6.x cluster even if the snapshot was created by a 2.x cluster.
+restored to a 6.x cluster even if the snapshot was created by a 5.x cluster.
 
 Elasticsearch nodes will fail to start if incompatible indices are present.
 
@@ -72,4 +72,4 @@ include::upgrade/rolling_upgrade.asciidoc[]
 
 include::upgrade/cluster_restart.asciidoc[]
 
-include::upgrade/reindex_upgrade.asciidoc[]
\ No newline at end of file
+include::upgrade/reindex_upgrade.asciidoc[]

From f3293879b519f050d522934335c6c581aa5b69c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= 
Date: Fri, 15 Dec 2017 10:45:44 +0100
Subject: [PATCH 275/297] [Docs] Improve rendering of ranking evaluation docs

---
 docs/reference/search/rank-eval.asciidoc | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc
index 5a3277fa43b..82a2df27d80 100644
--- a/docs/reference/search/rank-eval.asciidoc
+++ b/docs/reference/search/rank-eval.asciidoc
@@ -7,9 +7,9 @@ list or manually rated documents, the `_rank_eval` endpoint calculates and
 returns typical information retrieval metrics like _mean reciprocal rank_,
 _precision_ or _discounted cumulative gain_.
 
-experimental[The ranking evaluation API is new and may change in non-backwards compatible ways in the future,
-even on minor versions updates.]
+experimental[The ranking evaluation API is new and may change in non-backwards compatible ways in the future, even on minor versions updates.]
 
+[float]
 === Overview
 
 Search quality evaluation starts with looking at the users of your search application, and the things that they are searching for.
@@ -30,6 +30,7 @@ In order to get started with search quality evaluation, three basic things are n
 
 The ranking evaluation API provides a convenient way to use this information in a ranking evaluation request to calculate different search evaluation metrics. This gives a first estimation of your overall search quality and give you a measurement to optimize against when fine-tuning various aspect of the query generation in your application. 
 
+[float]
 === Ranking evaluation request structure
 
 In its most basic form, a request to the `_rank_eval` endpoint has two sections:
@@ -86,6 +87,7 @@ the rating of the documents relevance with regards to this search request
 
 A document `rating` can be any integer value that expresses the relevance of the document on a user defined scale. For some of the metrics, just giving a binary rating (e.g. `0` for irrelevant and `1` for relevant) will be sufficient, other metrics can use a more fine grained scale.  
 
+[float]
 === Template based ranking evaluation
 
 As an alternative to having to provide a single query per test request, it is possible to specify query templates in the evaluation request and later refer to them. Queries with similar structure that only differ in their parameters don't have to be repeated all the time in the `requests` section this way. In typical search systems where user inputs usually get filled into a small set of query templates, this helps making the evaluation request more succinct.
@@ -127,11 +129,13 @@ GET /my_index/_rank_eval
 <3> a reference to a previously defined temlate
 <4> the parameters to use to fill the template
 
+[float]
 === Available evaluation metrics
 
 The `metric` section determines which of the available evaluation metrics is going to be used.
 Currently, the following metrics are supported:
 
+[float]
 ==== Precision at K (P@k)
 
 This metric measures the number of relevant results in the top k search results. Its a form of the well known https://en.wikipedia.org/wiki/Information_retrieval#Precision[Precision] metric that only looks at the top k documents. It is the fraction of relevant documents in those first k
@@ -174,6 +178,7 @@ The `precision` metric takes the following optional parameters
 If set to 'true', unlabeled documents are ignored and neither count as relevant or irrelevant. Set to 'false' (the default), they are treated as irrelevant.
 |=======================================================================
 
+[float]
 ==== Mean reciprocal rank
 
 For every query in the test suite, this metric calculates the reciprocal of the rank of the
@@ -210,6 +215,7 @@ The `mean_reciprocal_rank` metric takes the following optional parameters
 "relevant". Defaults to `1`.
 |=======================================================================
 
+[float]
 ==== Discounted cumulative gain (DCG)
 
 In contrast to the two metrics above, https://en.wikipedia.org/wiki/Discounted_cumulative_gain[discounted cumulative gain] takes both, the rank and the rating of the search results, into account.
@@ -244,6 +250,7 @@ The `dcg` metric takes the following optional parameters:
 |`normalize` | If set to `true`, this metric will calculate the https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG[Normalized DCG].
 |=======================================================================
 
+[float]
 === Response format
 
 The response of the `_rank_eval` endpoint contains the overall calculated result for the defined quality metric, 

From cef7bd2079c15995b73f9e514a66d5d618bcb400 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen 
Date: Thu, 14 Dec 2017 20:10:04 +0100
Subject: [PATCH 276/297] docs: add best practises for wildcard queries inside
 percolator queries

---
 .../mapping/types/percolator.asciidoc         | 280 ++++++++++++++++++
 1 file changed, 280 insertions(+)

diff --git a/docs/reference/mapping/types/percolator.asciidoc b/docs/reference/mapping/types/percolator.asciidoc
index 35a4d7d6b75..b5226b53ba0 100644
--- a/docs/reference/mapping/types/percolator.asciidoc
+++ b/docs/reference/mapping/types/percolator.asciidoc
@@ -423,6 +423,286 @@ This results in a response like this:
 --------------------------------------------------
 // TESTRESPONSE[s/"took": 6,/"took": "$body.took",/]
 
+[float]
+==== Optimizing wildcard queries.
+
+Wildcard queries are  more expensive than other queries for the percolator,
+especially if the wildcard expressions are large.
+
+In the case of `wildcard` queries with prefix wildcard expressions or just the `prefix` query,
+the `edge_ngram` token filter can be used to replace these queries with regular `term`
+query on a field where the `edge_ngram` token filter is configured.
+
+Creating an index with custom analysis settings:
+
+[source,js]
+--------------------------------------------------
+PUT my_queries1
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "wildcard_prefix": { <1>
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": [
+            "standard",
+            "lowercase",
+            "wildcard_edge_ngram"
+          ]
+        }
+      },
+      "filter": {
+        "wildcard_edge_ngram": { <2>
+          "type": "edge_ngram",
+          "min_gram": 1,
+          "max_gram": 32
+        }
+      }
+    }
+  },
+  "mappings": {
+    "query": {
+      "properties": {
+        "query": {
+          "type": "percolator"
+        },
+        "my_field": {
+          "type": "text",
+          "fields": {
+            "prefix": { <3>
+              "type": "text",
+              "analyzer": "wildcard_prefix",
+              "search_analyzer": "standard"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+<1> The analyzer that generates the prefix tokens to be used at index time only.
+<2> Increase the `min_gram` and decrease `max_gram` settings based on your prefix search needs.
+<3> This multifield should be used to do the prefix search
+    with a `term` or `match` query instead of a `prefix` or `wildcard` query.
+
+
+Then instead of indexing the following query:
+
+[source,js]
+--------------------------------------------------
+{
+  "query": {
+    "wildcard": {
+      "my_field": "abc*"
+    }
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+this query below should be indexed:
+
+[source,js]
+--------------------------------------------------
+PUT /my_queries1/query/1?refresh
+{
+  "query": {
+    "term": {
+      "my_field.prefix": "abc"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+This way can handle the second query more efficiently than the first query.
+
+The following search request will match with the previously indexed
+percolator query:
+
+[source,js]
+--------------------------------------------------
+GET /my_queries1/_search
+{
+  "query": {
+    "percolate": {
+      "field": "query",
+      "document": {
+        "my_field": "abcd"
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+[source,js]
+--------------------------------------------------
+{
+  "took": 6,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "skipped": 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": 1,
+    "max_score": 0.41501677,
+    "hits": [
+      {
+        "_index": "my_queries1",
+        "_type": "query",
+        "_id": "1",
+        "_score": 0.41501677,
+        "_source": {
+          "query": {
+            "term": {
+              "my_field.prefix": "abc"
+            }
+          }
+        },
+        "fields": {
+          "_percolator_document_slot": [
+            0
+          ]
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took": 6,/"took": "$body.took",/]
+
+The same technique can also be used to speed up suffix
+wildcard searches. By using the `reverse` token filter
+before the `edge_ngram` token filter.
+
+[source,js]
+--------------------------------------------------
+PUT my_queries2
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "wildcard_suffix": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": [
+            "standard",
+            "lowercase",
+            "reverse",
+            "wildcard_edge_ngram"
+          ]
+        },
+        "wildcard_suffix_search_time": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": [
+            "standard",
+            "lowercase",
+            "reverse"
+          ]
+        }
+      },
+      "filter": {
+        "wildcard_edge_ngram": {
+          "type": "edge_ngram",
+          "min_gram": 1,
+          "max_gram": 32
+        }
+      }
+    }
+  },
+  "mappings": {
+    "query": {
+      "properties": {
+        "query": {
+          "type": "percolator"
+        },
+        "my_field": {
+          "type": "text",
+          "fields": {
+            "suffix": {
+              "type": "text",
+              "analyzer": "wildcard_suffix",
+              "search_analyzer": "wildcard_suffix_search_time" <1>
+            }
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+<1> A custom analyzer is needed at search time too, because otherwise
+    the query terms are not being reversed and would otherwise not match
+    with the reserved suffix tokens.
+
+Then instead of indexing the following query:
+
+[source,js]
+--------------------------------------------------
+{
+  "query": {
+    "wildcard": {
+      "my_field": "*xyz"
+    }
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+the following query below should be indexed:
+
+[source,js]
+--------------------------------------------------
+PUT /my_queries2/query/2?refresh
+{
+  "query": {
+    "match": { <1>
+      "my_field.suffix": "xyz"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+<1> The `match` query should be used instead of the `term` query,
+    because text analysis needs to reverse the query terms.
+
+The following search request will match with the previously indexed
+percolator query:
+
+[source,js]
+--------------------------------------------------
+GET /my_queries2/_search
+{
+  "query": {
+    "percolate": {
+      "field": "query",
+      "document": {
+        "my_field": "wxyz"
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
 [float]
 ==== Dedicated Percolator Index
 

From d941c64edb08c62638908c40f35c724dc396d91b Mon Sep 17 00:00:00 2001
From: Simon Willnauer 
Date: Fri, 15 Dec 2017 12:13:10 +0100
Subject: [PATCH 277/297] Optimize version map for append-only indexing
 (#27752)

Today we still maintain a version map even if we only index append-only
or in other words, documents with auto-generated IDs. We can instead maintain
an un-safe version map that will be swapped to a safe version map only if necessary
once we see the first document that requires access to the version map. For instance:
 * a auto-generated id retry
 * any kind of deletes
 * a document with a foreign ID (non-autogenerated

In these cases we forcefully refresh then internal reader and start maintaining
a version map until such a safe map wasn't necessary for two refresh cycles.
Indices / shards that never see an autogenerated ID document will always meintain a version
map and in the case of a delete / retry in a pure append-only index the version map will be
de-optimized for a short amount of time until we know it's safe again to swap back. This
will also minimize the requried refeshes.

Closes #19813
---
 .../index/engine/InternalEngine.java          |  40 ++++-
 .../index/engine/LiveVersionMap.java          | 157 ++++++++++++++++--
 .../index/engine/InternalEngineTests.java     | 119 +++++++++++++
 .../index/engine/LiveVersionMapTests.java     |  55 ++++++
 4 files changed, 351 insertions(+), 20 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index e4f6a6f9b0a..1a6dba8eb17 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -562,7 +562,7 @@ public class InternalEngine extends Engine {
             ensureOpen();
             SearcherScope scope;
             if (get.realtime()) {
-                VersionValue versionValue = versionMap.getUnderLock(get.uid().bytes());
+                VersionValue versionValue = getVersionFromMap(get.uid().bytes());
                 if (versionValue != null) {
                     if (versionValue.isDelete()) {
                         return GetResult.NOT_EXISTS;
@@ -600,7 +600,7 @@ public class InternalEngine extends Engine {
     private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnSeqNo(final Operation op) throws IOException {
         assert op.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "resolving ops based on seq# but no seqNo is found";
         final OpVsLuceneDocStatus status;
-        final VersionValue versionValue = versionMap.getUnderLock(op.uid().bytes());
+        VersionValue versionValue = getVersionFromMap(op.uid().bytes());
         assert incrementVersionLookup();
         if (versionValue != null) {
             if  (op.seqNo() > versionValue.seqNo ||
@@ -637,7 +637,7 @@ public class InternalEngine extends Engine {
     /** resolves the current version of the document, returning null if not found */
     private VersionValue resolveDocVersion(final Operation op) throws IOException {
         assert incrementVersionLookup(); // used for asserting in tests
-        VersionValue versionValue = versionMap.getUnderLock(op.uid().bytes());
+        VersionValue versionValue = getVersionFromMap(op.uid().bytes());
         if (versionValue == null) {
             assert incrementIndexVersionLookup(); // used for asserting in tests
             final long currentVersion = loadCurrentVersionFromIndex(op.uid());
@@ -651,6 +651,21 @@ public class InternalEngine extends Engine {
         return versionValue;
     }
 
+    private VersionValue getVersionFromMap(BytesRef id) {
+        if (versionMap.isUnsafe()) {
+            synchronized (versionMap) {
+                // we are switching from an unsafe map to a safe map. This might happen concurrently
+                // but we only need to do this once since the last operation per ID is to add to the version
+                // map so once we pass this point we can safely lookup from the version map.
+                if (versionMap.isUnsafe()) {
+                    refresh("unsafe_version_map", SearcherScope.INTERNAL);
+                }
+                versionMap.enforceSafeAccess();
+            }
+        }
+        return versionMap.getUnderLock(id);
+    }
+
     private boolean canOptimizeAddDocument(Index index) {
         if (index.getAutoGeneratedIdTimestamp() != IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP) {
             assert index.getAutoGeneratedIdTimestamp() >= 0 : "autoGeneratedIdTimestamp must be positive but was: "
@@ -812,6 +827,7 @@ public class InternalEngine extends Engine {
             assert index.version() == 1L : "can optimize on replicas but incoming version is [" + index.version() + "]";
             plan = IndexingStrategy.optimizedAppendOnly(index.seqNo());
         } else {
+            versionMap.enforceSafeAccess();
             // drop out of order operations
             assert index.versionType().versionTypeForReplicationAndRecovery() == index.versionType() :
                 "resolving out of order delivery based on versioning but version type isn't fit for it. got [" + index.versionType() + "]";
@@ -849,10 +865,12 @@ public class InternalEngine extends Engine {
         if (canOptimizeAddDocument(index)) {
             if (mayHaveBeenIndexedBefore(index)) {
                 plan = IndexingStrategy.overrideExistingAsIfNotThere(generateSeqNoForOperation(index), 1L);
+                versionMap.enforceSafeAccess();
             } else {
                 plan = IndexingStrategy.optimizedAppendOnly(generateSeqNoForOperation(index));
             }
         } else {
+            versionMap.enforceSafeAccess();
             // resolves incoming version
             final VersionValue versionValue = resolveDocVersion(index);
             final long currentVersion;
@@ -898,7 +916,7 @@ public class InternalEngine extends Engine {
                 assert assertDocDoesNotExist(index, canOptimizeAddDocument(index) == false);
                 index(index.docs(), indexWriter);
             }
-            versionMap.putUnderLock(index.uid().bytes(),
+            versionMap.maybePutUnderLock(index.uid().bytes(),
                 new VersionValue(plan.versionForIndexing, plan.seqNoForIndexing, index.primaryTerm()));
             return new IndexResult(plan.versionForIndexing, plan.seqNoForIndexing, plan.currentNotFoundOrDeleted);
         } catch (Exception ex) {
@@ -1018,7 +1036,9 @@ public class InternalEngine extends Engine {
      * Asserts that the doc in the index operation really doesn't exist
      */
     private boolean assertDocDoesNotExist(final Index index, final boolean allowDeleted) throws IOException {
-        final VersionValue versionValue = versionMap.getUnderLock(index.uid().bytes());
+        // NOTE this uses direct access to the version map since we are in the assertion code where we maintain a secondary
+        // map in the version map such that we don't need to refresh if we are unsafe;
+        final VersionValue versionValue = versionMap.getVersionForAssert(index.uid().bytes());
         if (versionValue != null) {
             if (versionValue.isDelete() == false || allowDeleted == false) {
                 throw new AssertionError("doc [" + index.type() + "][" + index.id() + "] exists in version map (version " + versionValue + ")");
@@ -1044,6 +1064,7 @@ public class InternalEngine extends Engine {
 
     @Override
     public DeleteResult delete(Delete delete) throws IOException {
+        versionMap.enforceSafeAccess();
         assert Objects.equals(delete.uid().field(), uidField) : delete.uid().field();
         assert assertVersionType(delete);
         assert assertIncomingSequenceNumber(delete.origin(), delete.seqNo());
@@ -2114,6 +2135,15 @@ public class InternalEngine extends Engine {
         return true;
     }
 
+    int getVersionMapSize() {
+        return versionMap.getAllCurrent().size();
+    }
+
+    boolean isSafeAccessRequired() {
+        return versionMap.isSafeAccessRequired();
+    }
+
+
     /**
      * Returns true iff the index writer has any deletions either buffered in memory or
      * in the index.
diff --git a/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java b/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
index 48d57ee7eec..5d58081b624 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
@@ -32,7 +32,7 @@ import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
 /** Maps _uid value to its version information. */
-class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
+final class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
 
     /**
      * Resets the internal map and adjusts it's capacity as if there were no indexing operations.
@@ -46,22 +46,100 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
         maps = new Maps();
     }
 
-    private static class Maps {
+    private static final class VersionLookup {
+
+        private static final VersionLookup EMPTY = new VersionLookup(Collections.emptyMap());
+        private final Map map;
+
+        // each version map has a notion of safe / unsafe which allows us to apply certain optimization in the auto-generated ID usecase
+        // where we know that documents can't have any duplicates so we can skip the version map entirely. This reduces
+        // the memory pressure significantly for this use-case where we often get a massive amount of small document (metrics).
+        // if the version map is in safeAccess mode we track all version in the version map. yet if a document comes in that needs
+        // safe access but we are not in this mode we force a refresh and make the map as safe access required. All subsequent ops will
+        // respect that and fill the version map. The nice part here is that we are only really requiring this for a single ID and since
+        // we hold the ID lock in the engine while we do all this it's safe to do it globally unlocked.
+        // NOTE: these values can both be non-volatile since it's ok to read a stale value per doc ID. We serialize changes in the engine
+        // that will prevent concurrent updates to the same document ID and therefore we can rely on the happens-before guanratee of the
+        // map reference itself.
+        private boolean unsafe;
+
+        private VersionLookup(Map map) {
+            this.map = map;
+        }
+
+        VersionValue get(BytesRef key) {
+            return map.get(key);
+        }
+
+        VersionValue put(BytesRef key, VersionValue value) {
+            return map.put(key, value);
+        }
+
+        boolean isEmpty() {
+            return map.isEmpty();
+        }
+
+
+        int size() {
+            return map.size();
+        }
+
+        boolean isUnsafe() {
+            return unsafe;
+        }
+
+        void markAsUnsafe() {
+            unsafe = true;
+        }
+    }
+
+    private static final class Maps {
 
         // All writes (adds and deletes) go into here:
-        final Map current;
+        final VersionLookup current;
 
         // Used while refresh is running, and to hold adds/deletes until refresh finishes.  We read from both current and old on lookup:
-        final Map old;
+        final VersionLookup old;
 
-        Maps(Map current, Map old) {
-           this.current = current;
-           this.old = old;
+        // this is not volatile since we don't need to maintain a happens before relation ship across doc IDs so it's enough to
+        // have the volatile read of the Maps reference to make it visible even across threads.
+        boolean needsSafeAccess;
+        final boolean previousMapsNeededSafeAccess;
+
+        Maps(VersionLookup current, VersionLookup old, boolean previousMapsNeededSafeAccess) {
+            this.current = current;
+            this.old = old;
+            this.previousMapsNeededSafeAccess = previousMapsNeededSafeAccess;
         }
 
         Maps() {
-            this(ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency(),
-                 Collections.emptyMap());
+            this(new VersionLookup(ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency()), VersionLookup.EMPTY, false);
+        }
+
+        boolean isSafeAccessMode() {
+            return needsSafeAccess || previousMapsNeededSafeAccess;
+        }
+
+        boolean shouldInheritSafeAccess() {
+            final boolean mapHasNotSeenAnyOperations = current.isEmpty() && current.isUnsafe() == false;
+            return needsSafeAccess
+                // we haven't seen any ops and map before needed it so we maintain it
+                || (mapHasNotSeenAnyOperations && previousMapsNeededSafeAccess);
+        }
+
+        /**
+         * Builds a new map for the refresh transition this should be called in beforeRefresh()
+         */
+        Maps buildTransitionMap() {
+            return new Maps(new VersionLookup(ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency(current.size())),
+                current, shouldInheritSafeAccess());
+        }
+
+        /**
+         * builds a new map that invalidates the old map but maintains the current. This should be called in afterRefresh()
+         */
+        Maps invalidateOldMap() {
+            return new Maps(current, VersionLookup.EMPTY, previousMapsNeededSafeAccess);
         }
     }
 
@@ -69,6 +147,9 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
     private final Map tombstones = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
 
     private volatile Maps maps = new Maps();
+    // we maintain a second map that only receives the updates that we skip on the actual map (unsafe ops)
+    // this map is only maintained if assertions are enabled
+    private volatile Maps unsafeKeysMap = new Maps();
 
     /** Bytes consumed for each BytesRef UID:
      * In this base value, we account for the {@link BytesRef} object itself as
@@ -113,8 +194,8 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
         // map.  While reopen is running, any lookup will first
         // try this new map, then fallback to old, then to the
         // current searcher:
-        maps = new Maps(ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency(maps.current.size()), maps.current);
-
+        maps = maps.buildTransitionMap();
+        assert (unsafeKeysMap = unsafeKeysMap.buildTransitionMap()) != null;
         // This is not 100% correct, since concurrent indexing ops can change these counters in between our execution of the previous
         // line and this one, but that should be minor, and the error won't accumulate over time:
         ramBytesUsedCurrent.set(0);
@@ -128,13 +209,18 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
         // case.  This is because we assign new maps (in beforeRefresh) slightly before Lucene actually flushes any segments for the
         // reopen, and so any concurrent indexing requests can still sneak in a few additions to that current map that are in fact reflected
         // in the previous reader.   We don't touch tombstones here: they expire on their own index.gc_deletes timeframe:
-        maps = new Maps(maps.current, Collections.emptyMap());
+
+        maps = maps.invalidateOldMap();
+        assert (unsafeKeysMap = unsafeKeysMap.invalidateOldMap()) != null;
+
     }
 
     /** Returns the live version (add or delete) for this uid. */
     VersionValue getUnderLock(final BytesRef uid) {
-        Maps currentMaps = maps;
+        return getUnderLock(uid, maps);
+    }
 
+    private VersionValue getUnderLock(final BytesRef uid, Maps currentMaps) {
         // First try to get the "live" value:
         VersionValue value = currentMaps.current.get(uid);
         if (value != null) {
@@ -149,11 +235,52 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
         return tombstones.get(uid);
     }
 
+    VersionValue getVersionForAssert(final BytesRef uid) {
+        VersionValue value = getUnderLock(uid, maps);
+        if (value == null) {
+            value = getUnderLock(uid, unsafeKeysMap);
+        }
+        return value;
+    }
+
+    boolean isUnsafe() {
+        return maps.current.isUnsafe() || maps.old.isUnsafe();
+    }
+
+    void enforceSafeAccess() {
+        maps.needsSafeAccess = true;
+    }
+
+    boolean isSafeAccessRequired() {
+        return maps.isSafeAccessMode();
+    }
+
+    /** Adds this uid/version to the pending adds map iff the map needs safe access.  */
+    void maybePutUnderLock(BytesRef uid, VersionValue version) {
+        Maps maps = this.maps;
+        if (maps.isSafeAccessMode()) {
+            putUnderLock(uid, version, maps);
+        } else {
+            maps.current.markAsUnsafe();
+            assert putAssertionMap(uid, version);
+        }
+    }
+
+    private boolean putAssertionMap(BytesRef uid, VersionValue version) {
+        putUnderLock(uid, version, unsafeKeysMap);
+        return true;
+    }
+
     /** Adds this uid/version to the pending adds map. */
     void putUnderLock(BytesRef uid, VersionValue version) {
+        Maps maps = this.maps;
+        putUnderLock(uid, version, maps);
+    }
+
+    /** Adds this uid/version to the pending adds map. */
+    private void putUnderLock(BytesRef uid, VersionValue version, Maps maps) {
         assert uid.bytes.length == uid.length : "Oversized _uid! UID length: " + uid.length + ", bytes length: " + uid.bytes.length;
         long uidRAMBytesUsed = BASE_BYTES_PER_BYTESREF + uid.bytes.length;
-
         final VersionValue prev = maps.current.put(uid, version);
         if (prev != null) {
             // Deduct RAM for the version we just replaced:
@@ -264,5 +391,5 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
 
     /** Returns the current internal versions as a point in time snapshot*/
     Map getAllCurrent() {
-        return maps.current;
+        return maps.current.map;
     }}
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 999b4dca563..7e73228ecfd 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -106,6 +106,7 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.RootObjectMapper;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
@@ -175,6 +176,69 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class InternalEngineTests extends EngineTestCase {
 
+    public void testVersionMapAfterAutoIDDocument() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField("test"),
+            new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        Engine.Index operation = randomBoolean() ?
+            appendOnlyPrimary(doc, false, 1)
+            : appendOnlyReplica(doc, false, 1, randomIntBetween(0, 5));
+        engine.index(operation);
+        assertFalse(engine.isSafeAccessRequired());
+        doc = testParsedDocument("1", null, testDocumentWithTextField("updated"),
+            new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        Engine.Index update = indexForDoc(doc);
+        engine.index(update);
+        assertTrue(engine.isSafeAccessRequired());
+        assertEquals(1, engine.getVersionMapSize());
+        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
+            assertEquals(0, searcher.reader().numDocs());
+        }
+
+        try (Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
+            assertEquals(1, searcher.reader().numDocs());
+            TopDocs search = searcher.searcher().search(new MatchAllDocsQuery(), 1);
+            org.apache.lucene.document.Document luceneDoc = searcher.searcher().doc(search.scoreDocs[0].doc);
+            assertEquals("test", luceneDoc.get("value"));
+        }
+
+        // now lets make this document visible
+        engine.refresh("test");
+        if (randomBoolean()) { // random empty refresh
+            engine.refresh("test");
+        }
+        assertTrue("safe access should be required we carried it over", engine.isSafeAccessRequired());
+        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
+            assertEquals(1, searcher.reader().numDocs());
+            TopDocs search = searcher.searcher().search(new MatchAllDocsQuery(), 1);
+            org.apache.lucene.document.Document luceneDoc = searcher.searcher().doc(search.scoreDocs[0].doc);
+            assertEquals("updated", luceneDoc.get("value"));
+        }
+
+        doc = testParsedDocument("2", null, testDocumentWithTextField("test"),
+            new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        operation = randomBoolean() ?
+            appendOnlyPrimary(doc, false, 1)
+            : appendOnlyReplica(doc, false, 1, engine.seqNoService().generateSeqNo());
+        engine.index(operation);
+        assertTrue("safe access should be required", engine.isSafeAccessRequired());
+        assertEquals(1, engine.getVersionMapSize()); // now we add this to the map
+        engine.refresh("test");
+        if (randomBoolean()) { // randomly refresh here again
+            engine.refresh("test");
+        }
+        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
+            assertEquals(2, searcher.reader().numDocs());
+        }
+        assertFalse("safe access should NOT be required last indexing round was only append only", engine.isSafeAccessRequired());
+        engine.delete(new Engine.Delete(operation.type(), operation.id(), operation.uid()));
+        assertTrue("safe access should be required", engine.isSafeAccessRequired());
+        engine.refresh("test");
+        assertTrue("safe access should be required", engine.isSafeAccessRequired());
+        try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
+            assertEquals(1, searcher.reader().numDocs());
+        }
+    }
+
     public void testSegments() throws Exception {
         try (Store store = createStore();
              InternalEngine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) {
@@ -3364,6 +3428,7 @@ public class InternalEngineTests extends EngineTestCase {
         }
         Collections.shuffle(docs, random());
         CountDownLatch startGun = new CountDownLatch(thread.length);
+
         AtomicInteger offset = new AtomicInteger(-1);
         for (int i = 0; i < thread.length; i++) {
             thread[i] = new Thread() {
@@ -3375,6 +3440,7 @@ public class InternalEngineTests extends EngineTestCase {
                     } catch (InterruptedException e) {
                         throw new AssertionError(e);
                     }
+                    assertEquals(0, engine.getVersionMapSize());
                     int docOffset;
                     while ((docOffset = offset.incrementAndGet()) < docs.size()) {
                         try {
@@ -3387,6 +3453,9 @@ public class InternalEngineTests extends EngineTestCase {
             };
             thread[i].start();
         }
+        try (Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
+            assertEquals("unexpected refresh", 0, searcher.reader().maxDoc());
+        }
         for (int i = 0; i < thread.length; i++) {
             thread[i].join();
         }
@@ -4227,4 +4296,54 @@ public class InternalEngineTests extends EngineTestCase {
             }
         }
     }
+
+    public void testConcurrentAppendUpdateAndRefresh() throws InterruptedException, IOException {
+        int numDocs = scaledRandomIntBetween(100, 1000);
+        CountDownLatch latch = new CountDownLatch(2);
+        AtomicBoolean done = new AtomicBoolean(false);
+        AtomicInteger numDeletes = new AtomicInteger();
+        Thread thread = new Thread(() -> {
+           try {
+               latch.countDown();
+               latch.await();
+               for (int j = 0; j < numDocs; j++) {
+                   String docID = Integer.toString(j);
+                   ParsedDocument doc = testParsedDocument(docID, null, testDocumentWithTextField(),
+                       new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+                   Engine.Index operation = appendOnlyPrimary(doc, false, 1);
+                   engine.index(operation);
+                   if (rarely()) {
+                       engine.delete(new Engine.Delete(operation.type(), operation.id(), operation.uid()));
+                       numDeletes.incrementAndGet();
+                   } else {
+                       doc = testParsedDocument(docID, null, testDocumentWithTextField("updated"),
+                           new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+                       Engine.Index update = indexForDoc(doc);
+                       engine.index(update);
+                   }
+               }
+           } catch (Exception e) {
+               throw new AssertionError(e);
+           } finally {
+               done.set(true);
+           }
+        });
+        thread.start();
+        latch.countDown();
+        latch.await();
+        while (done.get() == false) {
+            engine.refresh("test", Engine.SearcherScope.INTERNAL);
+        }
+        thread.join();
+        engine.refresh("test", Engine.SearcherScope.INTERNAL);
+        try (Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
+            TopDocs search = searcher.searcher().search(new MatchAllDocsQuery(), searcher.reader().numDocs());
+            for (int i = 0; i < search.scoreDocs.length; i++) {
+                org.apache.lucene.document.Document luceneDoc = searcher.searcher().doc(search.scoreDocs[i].doc);
+                assertEquals("updated", luceneDoc.get("value"));
+            }
+            int totalNumDocs = numDocs - numDeletes.get();
+            assertEquals(totalNumDocs, searcher.reader().numDocs());
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java b/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
index 77e5b55ac57..f3613f72cd6 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
@@ -217,4 +217,59 @@ public class LiveVersionMapTests extends ESTestCase {
             assertTrue(versionValue instanceof DeleteVersionValue);
         });
     }
+
+    public void testCarryOnSafeAccess() throws IOException {
+        LiveVersionMap map = new LiveVersionMap();
+        assertFalse(map.isUnsafe());
+        assertFalse(map.isSafeAccessRequired());
+        map.enforceSafeAccess();
+        assertTrue(map.isSafeAccessRequired());
+        assertFalse(map.isUnsafe());
+        int numIters = randomIntBetween(1, 5);
+        for (int i = 0; i < numIters; i++) { // if we don't do anything ie. no adds etc we will stay with the safe access required
+            map.beforeRefresh();
+            map.afterRefresh(randomBoolean());
+            assertTrue("failed in iter: " + i, map.isSafeAccessRequired());
+        }
+
+        map.maybePutUnderLock(new BytesRef(""), new VersionValue(randomLong(), randomLong(), randomLong()));
+        assertFalse(map.isUnsafe());
+        assertEquals(1, map.getAllCurrent().size());
+
+        map.beforeRefresh();
+        map.afterRefresh(randomBoolean());
+        assertFalse(map.isUnsafe());
+        assertFalse(map.isSafeAccessRequired());
+
+        map.maybePutUnderLock(new BytesRef(""), new VersionValue(randomLong(), randomLong(), randomLong()));
+        assertTrue(map.isUnsafe());
+        assertFalse(map.isSafeAccessRequired());
+        assertEquals(0, map.getAllCurrent().size());
+    }
+
+    public void testRefreshTransition() throws IOException {
+        LiveVersionMap map = new LiveVersionMap();
+        map.maybePutUnderLock(uid("1"), new VersionValue(randomLong(), randomLong(), randomLong()));
+        assertTrue(map.isUnsafe());
+        assertNull(map.getUnderLock(uid("1")));
+        map.beforeRefresh();
+        assertTrue(map.isUnsafe());
+        assertNull(map.getUnderLock(uid("1")));
+        map.afterRefresh(randomBoolean());
+        assertNull(map.getUnderLock(uid("1")));
+        assertFalse(map.isUnsafe());
+
+        map.enforceSafeAccess();
+        map.maybePutUnderLock(uid("1"), new VersionValue(randomLong(), randomLong(), randomLong()));
+        assertFalse(map.isUnsafe());
+        assertNotNull(map.getUnderLock(uid("1")));
+        map.beforeRefresh();
+        assertFalse(map.isUnsafe());
+        assertTrue(map.isSafeAccessRequired());
+        assertNotNull(map.getUnderLock(uid("1")));
+        map.afterRefresh(randomBoolean());
+        assertNull(map.getUnderLock(uid("1")));
+        assertFalse(map.isUnsafe());
+        assertTrue(map.isSafeAccessRequired());
+    }
 }

From c93cc1bb8f5003184f945db0cc4d157b2f1ba70f Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe 
Date: Fri, 15 Dec 2017 12:09:59 +0000
Subject: [PATCH 278/297] Fix ByteSizeValue serialisation test

---
 .../java/org/elasticsearch/common/unit/ByteSizeValueTests.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
index d9010136ca0..7cf96b49bb0 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
@@ -208,7 +208,7 @@ public class ByteSizeValueTests extends AbstractWireSerializingTestCase= Long.MAX_VALUE / unit.toBytes(1)) {
+            if (size > Long.MAX_VALUE / unit.toBytes(1)) {
                 throw new AssertionError();
             }
             return new ByteSizeValue(size, unit);

From 481d98b8d5324d9bf4759dad59a09dbd7dd74e94 Mon Sep 17 00:00:00 2001
From: Simon Willnauer 
Date: Fri, 15 Dec 2017 15:20:55 +0100
Subject: [PATCH 279/297] Remove `operationThreaded` from Java API (#27836)

This option is completely unused. Some places set it but we never read
the value neither respect it.
---
 .../SingleShardOperationRequestBuilder.java   |  9 --------
 .../single/shard/SingleShardRequest.java      | 16 --------------
 .../shard/TransportSingleShardAction.java     |  1 -
 .../index/query/GeoShapeQueryBuilder.java     |  1 -
 .../rest/action/document/RestGetAction.java   |  1 -
 .../action/document/RestGetSourceAction.java  |  1 -
 .../document/DocumentActionsIT.java           |  4 ++--
 .../recovery/SimpleRecoveryIT.java            | 14 ++++++------
 docs/java-api/docs/delete.asciidoc            | 22 -------------------
 docs/java-api/docs/get.asciidoc               | 22 -------------------
 docs/java-api/docs/index_.asciidoc            | 13 -----------
 11 files changed, 9 insertions(+), 95 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java
index 03d4fb6343d..af6ee2287f2 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java
@@ -39,13 +39,4 @@ public abstract class SingleShardOperationRequestBuilder() {
                 @Override
                 public void onResponse(Response result) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index 0f9006ed7da..c4a6f7fa6b7 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -383,7 +383,6 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder(){
 
             @Override
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
index 3265a59692b..8f86d9ae600 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
@@ -56,7 +56,6 @@ public class RestGetAction extends BaseRestHandler {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final GetRequest getRequest = new GetRequest(request.param("index"), request.param("type"), request.param("id"));
-        getRequest.operationThreaded(true);
         getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh()));
         getRequest.routing(request.param("routing"));
         getRequest.parent(request.param("parent"));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
index fead47db744..ca1d20da777 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
@@ -59,7 +59,6 @@ public class RestGetSourceAction extends BaseRestHandler {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final GetRequest getRequest = new GetRequest(request.param("index"), request.param("type"), request.param("id"));
-        getRequest.operationThreaded(true);
         getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh()));
         getRequest.routing(request.param("routing"));
         getRequest.parent(request.param("parent"));
diff --git a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
index c0011f79026..88bab97a1f3 100644
--- a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
+++ b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
@@ -95,11 +95,11 @@ public class DocumentActionsIT extends ESIntegTestCase {
 
         logger.info("Get [type1/1]");
         for (int i = 0; i < 5; i++) {
-            getResult = client().prepareGet("test", "type1", "1").setOperationThreaded(false).execute().actionGet();
+            getResult = client().prepareGet("test", "type1", "1").execute().actionGet();
             assertThat(getResult.getIndex(), equalTo(getConcreteIndexName()));
             assertThat("cycle #" + i, getResult.getSourceAsString(), equalTo(source("1", "test").string()));
             assertThat("cycle(map) #" + i, (String) getResult.getSourceAsMap().get("name"), equalTo("test"));
-            getResult = client().get(getRequest("test").type("type1").id("1").operationThreaded(true)).actionGet();
+            getResult = client().get(getRequest("test").type("type1").id("1")).actionGet();
             assertThat("cycle #" + i, getResult.getSourceAsString(), equalTo(source("1", "test").string()));
             assertThat(getResult.getIndex(), equalTo(getConcreteIndexName()));
         }
diff --git a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java
index bf8bce8ae63..3096c7b9e4f 100644
--- a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java
@@ -72,13 +72,13 @@ public class SimpleRecoveryIT extends ESIntegTestCase {
         GetResponse getResult;
 
         for (int i = 0; i < 5; i++) {
-            getResult = client().get(getRequest("test").type("type1").id("1").operationThreaded(false)).actionGet();
+            getResult = client().get(getRequest("test").type("type1").id("1")).actionGet();
             assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
-            getResult = client().get(getRequest("test").type("type1").id("1").operationThreaded(false)).actionGet();
+            getResult = client().get(getRequest("test").type("type1").id("1")).actionGet();
             assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
-            getResult = client().get(getRequest("test").type("type1").id("2").operationThreaded(true)).actionGet();
+            getResult = client().get(getRequest("test").type("type1").id("2")).actionGet();
             assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
-            getResult = client().get(getRequest("test").type("type1").id("2").operationThreaded(true)).actionGet();
+            getResult = client().get(getRequest("test").type("type1").id("2")).actionGet();
             assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
         }
 
@@ -95,11 +95,11 @@ public class SimpleRecoveryIT extends ESIntegTestCase {
             assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
             getResult = client().get(getRequest("test").type("type1").id("1")).actionGet();
             assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
-            getResult = client().get(getRequest("test").type("type1").id("2").operationThreaded(true)).actionGet();
+            getResult = client().get(getRequest("test").type("type1").id("2")).actionGet();
             assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
-            getResult = client().get(getRequest("test").type("type1").id("2").operationThreaded(true)).actionGet();
+            getResult = client().get(getRequest("test").type("type1").id("2")).actionGet();
             assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
-            getResult = client().get(getRequest("test").type("type1").id("2").operationThreaded(true)).actionGet();
+            getResult = client().get(getRequest("test").type("type1").id("2")).actionGet();
             assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
         }
     }
diff --git a/docs/java-api/docs/delete.asciidoc b/docs/java-api/docs/delete.asciidoc
index 3e3788cda24..218ea14553b 100644
--- a/docs/java-api/docs/delete.asciidoc
+++ b/docs/java-api/docs/delete.asciidoc
@@ -14,28 +14,6 @@ DeleteResponse response = client.prepareDelete("twitter", "tweet", "1").get();
 For more information on the delete operation, check out the
 {ref}/docs-delete.html[delete API] docs.
 
-
-[[java-docs-delete-thread]]
-==== Operation Threading
-
-The delete API allows to set the threading model the operation will be
-performed when the actual execution of the API is performed on the same
-node (the API is executed on a shard that is allocated on the same
-server).
-
-The options are to execute the operation on a different thread, or to
-execute it on the calling thread (note that the API is still async). By
-default, `operationThreaded` is set to `true` which means the operation
-is executed on a different thread. Here is an example that sets it to
-`false`:
-
-[source,java]
---------------------------------------------------
-DeleteResponse response = client.prepareDelete("twitter", "tweet", "1")
-        .setOperationThreaded(false)
-        .get();
---------------------------------------------------
-
 [[java-docs-delete-by-query]]
 === Delete By Query API
 
diff --git a/docs/java-api/docs/get.asciidoc b/docs/java-api/docs/get.asciidoc
index cde1b55a077..24f17e1c9ad 100644
--- a/docs/java-api/docs/get.asciidoc
+++ b/docs/java-api/docs/get.asciidoc
@@ -12,25 +12,3 @@ GetResponse response = client.prepareGet("twitter", "tweet", "1").get();
 
 For more information on the get operation, check out the REST
 {ref}/docs-get.html[get] docs.
-
-
-[[java-docs-get-thread]]
-==== Operation Threading
-
-The get API allows to set the threading model the operation will be
-performed when the actual execution of the API is performed on the same
-node (the API is executed on a shard that is allocated on the same
-server).
-
-The options are to execute the operation on a different thread, or to
-execute it on the calling thread (note that the API is still async). By
-default, `operationThreaded` is set to `true` which means the operation
-is executed on a different thread. Here is an example that sets it to
-`false`:
-
-[source,java]
---------------------------------------------------
-GetResponse response = client.prepareGet("twitter", "tweet", "1")
-        .setOperationThreaded(false)
-        .get();
---------------------------------------------------
diff --git a/docs/java-api/docs/index_.asciidoc b/docs/java-api/docs/index_.asciidoc
index b32955d9d4f..b455a7ab01f 100644
--- a/docs/java-api/docs/index_.asciidoc
+++ b/docs/java-api/docs/index_.asciidoc
@@ -163,16 +163,3 @@ RestStatus status = response.status();
 For more information on the index operation, check out the REST
 {ref}/docs-index_.html[index] docs.
 
-
-[[java-docs-index-thread]]
-==== Operation Threading
-
-The index API allows one to set the threading model the operation will be
-performed when the actual execution of the API is performed on the same
-node (the API is executed on a shard that is allocated on the same
-server).
-
-The options are to execute the operation on a different thread, or to
-execute it on the calling thread (note that the API is still asynchronous). By
-default, `operationThreaded` is set to `true` which means the operation
-is executed on a different thread.

From 7945848dd6b8adbb6790c262a78a2d66f85ff877 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Fri, 15 Dec 2017 10:56:00 -0500
Subject: [PATCH 280/297] Register HTTP read timeout setting

This commit registers the HTTP read timeout setting so that it can
actually be set.
---
 .../java/org/elasticsearch/common/settings/ClusterSettings.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index ae28b42cf16..db8dd461dd7 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -246,6 +246,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE,
                     HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE,
                     HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH,
+                    HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT,
                     HttpTransportSettings.SETTING_HTTP_RESET_COOKIES,
                     HttpTransportSettings.SETTING_HTTP_TCP_NO_DELAY,
                     HttpTransportSettings.SETTING_HTTP_TCP_KEEP_ALIVE,

From e542c03bea6f93cb6bfaa97e8bdbda5fcb2567eb Mon Sep 17 00:00:00 2001
From: sandstrom 
Date: Fri, 15 Dec 2017 17:29:13 +0100
Subject: [PATCH 281/297] Clarify that number of threads is set by packages

This commit clarifies that on systemd the packages already set the number of
threads for the Elasticsearch process.

Relates #27840
---
 docs/reference/setup/sysconfig/threads.asciidoc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/reference/setup/sysconfig/threads.asciidoc b/docs/reference/setup/sysconfig/threads.asciidoc
index a9a75dc872b..0f6f1b31ece 100644
--- a/docs/reference/setup/sysconfig/threads.asciidoc
+++ b/docs/reference/setup/sysconfig/threads.asciidoc
@@ -2,7 +2,7 @@
 === Number of threads
 
 Elasticsearch uses a number of thread pools for different types of operations.
-It is important that it is able to create new threads whenever needed.   Make
+It is important that it is able to create new threads whenever needed. Make
 sure that the number of threads that the Elasticsearch user can create is at
 least 4096.
 
@@ -10,3 +10,6 @@ This can be done by setting <> as root before
 starting Elasticsearch, or by setting `nproc` to `4096` in
 <>.
 
+The package distributions when run as services under `systemd` will configure
+the number of threads for the Elasticsearch process automatically. No
+additional configuration is required.

From f5e0932c8ddc2cff808b60f04fca46de601f6260 Mon Sep 17 00:00:00 2001
From: kel 
Date: Sat, 16 Dec 2017 01:00:40 +0800
Subject: [PATCH 282/297] Add version support for inner hits in field
 collapsing (#27822) (#27833)

Add version support for inner hits in field collapsing
---
 .../action/search/ExpandSearchPhase.java      |  1 +
 .../action/search/ExpandSearchPhaseTests.java |  6 +-
 .../test/search/110_field_collapsing.yml      | 65 +++++++++++++++++++
 3 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
index 53ce4299c54..35df6212833 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
@@ -165,6 +165,7 @@ final class ExpandSearchPhase extends SearchPhase {
         }
         groupSource.explain(options.isExplain());
         groupSource.trackScores(options.isTrackScores());
+        groupSource.version(options.isVersion());
         return groupSource;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
index 0951380fcf4..b580d48c11a 100644
--- a/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
@@ -248,6 +247,8 @@ public class ExpandSearchPhaseTests extends ESTestCase {
 
     public void testExpandRequestOptions() throws IOException {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
+        boolean version = randomBoolean();
+
         mockSearchPhaseContext.searchTransport = new SearchTransportService(
             Settings.builder().put("search.remote.connect", false).build(), null, null) {
 
@@ -256,13 +257,14 @@ public class ExpandSearchPhaseTests extends ESTestCase {
                 final QueryBuilder postFilter = QueryBuilders.existsQuery("foo");
                 assertTrue(request.requests().stream().allMatch((r) -> "foo".equals(r.preference())));
                 assertTrue(request.requests().stream().allMatch((r) -> "baz".equals(r.routing())));
+                assertTrue(request.requests().stream().allMatch((r) -> version == r.source().version()));
                 assertTrue(request.requests().stream().allMatch((r) -> postFilter.equals(r.source().postFilter())));
             }
         };
         mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder()
             .collapse(
                 new CollapseBuilder("someField")
-                    .setInnerHits(new InnerHitBuilder().setName("foobarbaz"))
+                    .setInnerHits(new InnerHitBuilder().setName("foobarbaz").setVersion(version))
             )
             .postFilter(QueryBuilders.existsQuery("foo")))
             .preference("foobar")
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
index fa012ff3acb..48ca92c1ee9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
@@ -7,36 +7,48 @@ setup:
           index:  test
           type:   test
           id:     1
+          version_type: external
+          version: 11
           body:   { numeric_group: 1, sort: 10 }
   - do:
       index:
           index:  test
           type:   test
           id:     2
+          version_type: external
+          version: 22
           body:   { numeric_group: 1, sort: 6 }
   - do:
       index:
           index:  test
           type:   test
           id:     3
+          version_type: external
+          version: 33
           body:   { numeric_group: 1, sort: 24 }
   - do:
       index:
           index:  test
           type:   test
           id:     4
+          version_type: external
+          version: 44
           body:   { numeric_group: 25, sort: 10 }
   - do:
       index:
           index:  test
           type:   test
           id:     5
+          version_type: external
+          version: 55
           body:   { numeric_group: 25, sort: 5 }
   - do:
       index:
           index:  test
           type:   test
           id:     6
+          version_type: external
+          version: 66
           body:   { numeric_group: 3, sort: 36 }
   - do:
       indices.refresh:
@@ -322,3 +334,56 @@ setup:
   - match: { hits.hits.2.inner_hits.sub_hits_desc.hits.total: 2 }
   - length: { hits.hits.2.inner_hits.sub_hits_desc.hits.hits: 1 }
   - match: { hits.hits.2.inner_hits.sub_hits_desc.hits.hits.0._id: "4" }
+
+---
+"field collapsing, inner_hits and version":
+
+  - skip:
+      version: " - 6.99.99"
+      reason:  "bug fixed in 7.0.0"
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body:
+          collapse: { field: numeric_group, inner_hits: { name: sub_hits, version: true, size: 2, sort: [{ sort: asc }] } }
+          sort: [{ sort: desc }]
+          version: true
+
+  - match: { hits.total: 6 }
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0._type: test }
+  - match: { hits.hits.0.fields.numeric_group: [3] }
+  - match: { hits.hits.0.sort: [36] }
+  - match: { hits.hits.0._id: "6" }
+  - match: { hits.hits.0._version: 66 }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.total: 1 }
+  - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._version: 66 }
+  - match: { hits.hits.1._index: test }
+  - match: { hits.hits.1._type: test }
+  - match: { hits.hits.1.fields.numeric_group: [1] }
+  - match: { hits.hits.1.sort: [24] }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.1._version: 33 }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.total: 3 }
+  - length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._version: 22 }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._version: 11 }
+  - match: { hits.hits.2._index: test }
+  - match: { hits.hits.2._type: test }
+  - match: { hits.hits.2.fields.numeric_group: [25] }
+  - match: { hits.hits.2.sort: [10] }
+  - match: { hits.hits.2._id: "4" }
+  - match: { hits.hits.2._version: 44 }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.total: 2 }
+  - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._version: 55 }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._version: 44 }

From 916e7dbe29a5c989fa0e3df0593225c59b6fdfba Mon Sep 17 00:00:00 2001
From: Tim Brooks 
Date: Fri, 15 Dec 2017 10:42:00 -0700
Subject: [PATCH 283/297] Add NioGroup for use in different transports (#27737)

This commit is related to #27260. It adds a base NioGroup for use in
different transports. This class creates and starts the underlying
selectors. Different protocols or transports are established by passing
the ChannelFactory to the bindServerChannel or openChannel
methods. This allows a TcpChannelFactory to be passed which will
create and register channels that support the elasticsearch tcp binary
protocol or a channel factory that will create http channels (or other).
---
 .../elasticsearch/transport/nio/NioGroup.java | 129 ++++++++++++++++++
 .../transport/nio/NioShutdown.java            |  55 --------
 .../transport/nio/NioTransport.java           |  70 +++-------
 ...rSupplier.java => RoundRobinSupplier.java} |  13 +-
 .../nio/AcceptorEventHandlerTests.java        |   2 +-
 .../transport/nio/NioGroupTests.java          |  82 +++++++++++
 .../nio/SimpleNioTransportTests.java          |   4 +-
 7 files changed, 241 insertions(+), 114 deletions(-)
 create mode 100644 test/framework/src/main/java/org/elasticsearch/transport/nio/NioGroup.java
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/transport/nio/NioShutdown.java
 rename test/framework/src/main/java/org/elasticsearch/transport/nio/{RoundRobinSelectorSupplier.java => RoundRobinSupplier.java} (73%)
 create mode 100644 test/framework/src/test/java/org/elasticsearch/transport/nio/NioGroupTests.java

diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioGroup.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioGroup.java
new file mode 100644
index 00000000000..b0e1862c706
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioGroup.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport.nio;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.transport.nio.channel.ChannelFactory;
+import org.elasticsearch.transport.nio.channel.NioServerSocketChannel;
+import org.elasticsearch.transport.nio.channel.NioSocketChannel;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * The NioGroup is a group of selectors for interfacing with java nio. When it is started it will create the
+ * configured number of socket and acceptor selectors. Each selector will be running in a dedicated thread.
+ * Server connections can be bound using the {@link #bindServerChannel(InetSocketAddress, ChannelFactory)}
+ * method. Client connections can be opened using the {@link #openChannel(InetSocketAddress, ChannelFactory)}
+ * method.
+ * 
      
+ * The logic specific to a particular channel is provided by the {@link ChannelFactory} passed to the method
+ * when the channel is created. This is what allows an NioGroup to support different channel types.
+ */
+public class NioGroup implements AutoCloseable {
+
+
+    private final ArrayList acceptors;
+    private final RoundRobinSupplier acceptorSupplier;
+
+    private final ArrayList socketSelectors;
+    private final RoundRobinSupplier socketSelectorSupplier;
+
+    private final AtomicBoolean isOpen = new AtomicBoolean(true);
+
+    public NioGroup(Logger logger, ThreadFactory acceptorThreadFactory, int acceptorCount,
+                    BiFunction, AcceptorEventHandler> acceptorEventHandlerFunction,
+                    ThreadFactory socketSelectorThreadFactory, int socketSelectorCount,
+                    Function socketEventHandlerFunction) throws IOException {
+        acceptors = new ArrayList<>(acceptorCount);
+        socketSelectors = new ArrayList<>(socketSelectorCount);
+
+        try {
+            for (int i = 0; i < socketSelectorCount; ++i) {
+                SocketSelector selector = new SocketSelector(socketEventHandlerFunction.apply(logger));
+                socketSelectors.add(selector);
+            }
+            startSelectors(socketSelectors, socketSelectorThreadFactory);
+
+            for (int i = 0; i < acceptorCount; ++i) {
+                SocketSelector[] childSelectors = this.socketSelectors.toArray(new SocketSelector[this.socketSelectors.size()]);
+                Supplier selectorSupplier = new RoundRobinSupplier<>(childSelectors);
+                AcceptingSelector acceptor = new AcceptingSelector(acceptorEventHandlerFunction.apply(logger, selectorSupplier));
+                acceptors.add(acceptor);
+            }
+            startSelectors(acceptors, acceptorThreadFactory);
+        } catch (Exception e) {
+            try {
+                close();
+            } catch (Exception e1) {
+                e.addSuppressed(e1);
+            }
+            throw e;
+        }
+
+        socketSelectorSupplier = new RoundRobinSupplier<>(socketSelectors.toArray(new SocketSelector[socketSelectors.size()]));
+        acceptorSupplier = new RoundRobinSupplier<>(acceptors.toArray(new AcceptingSelector[acceptors.size()]));
+    }
+
+    public  S bindServerChannel(InetSocketAddress address, ChannelFactory factory)
+        throws IOException {
+        ensureOpen();
+        if (acceptors.isEmpty()) {
+            throw new IllegalArgumentException("There are no acceptors configured. Without acceptors, server channels are not supported.");
+        }
+        return factory.openNioServerSocketChannel(address, acceptorSupplier.get());
+    }
+
+    public  S openChannel(InetSocketAddress address, ChannelFactory factory) throws IOException {
+        ensureOpen();
+        return factory.openNioChannel(address, socketSelectorSupplier.get());
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (isOpen.compareAndSet(true, false)) {
+            IOUtils.close(Stream.concat(acceptors.stream(), socketSelectors.stream()).collect(Collectors.toList()));
+        }
+    }
+
+    private static  void startSelectors(Iterable selectors, ThreadFactory threadFactory) {
+        for (ESSelector acceptor : selectors) {
+            if (acceptor.isRunning() == false) {
+                threadFactory.newThread(acceptor::runLoop).start();
+                acceptor.isRunningFuture().actionGet();
+            }
+        }
+    }
+
+    private void ensureOpen() {
+        if (isOpen.get() == false) {
+            throw new IllegalStateException("NioGroup is closed.");
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioShutdown.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioShutdown.java
deleted file mode 100644
index 3970e69b2c1..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioShutdown.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.nio;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.ElasticsearchException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.concurrent.CountDownLatch;
-
-public class NioShutdown {
-
-    private final Logger logger;
-
-    public NioShutdown(Logger logger) {
-        this.logger = logger;
-    }
-
-    void orderlyShutdown(ArrayList acceptors, ArrayList socketSelectors) {
-
-        for (AcceptingSelector acceptor : acceptors) {
-            shutdownSelector(acceptor);
-        }
-
-        for (SocketSelector selector : socketSelectors) {
-            shutdownSelector(selector);
-        }
-    }
-
-    private void shutdownSelector(ESSelector selector) {
-        try {
-            selector.close();
-        } catch (IOException | ElasticsearchException e) {
-            logger.warn("unexpected exception while stopping selector", e);
-        }
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java
index 775030bc6db..bb28d93f85a 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/NioTransport.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.transport.nio;
 
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -46,9 +47,7 @@ import org.elasticsearch.transport.nio.channel.TcpWriteContext;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ThreadFactory;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 
@@ -71,11 +70,8 @@ public class NioTransport extends TcpTransport {
 
     private final PageCacheRecycler pageCacheRecycler;
     private final ConcurrentMap profileToChannelFactory = newConcurrentMap();
-    private final ArrayList acceptors = new ArrayList<>();
-    private final ArrayList socketSelectors = new ArrayList<>();
-    private RoundRobinSelectorSupplier clientSelectorSupplier;
-    private TcpChannelFactory clientChannelFactory;
-    private int acceptorNumber;
+    private volatile NioGroup nioGroup;
+    private volatile TcpChannelFactory clientChannelFactory;
 
     public NioTransport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays,
                         PageCacheRecycler pageCacheRecycler, NamedWriteableRegistry namedWriteableRegistry,
@@ -87,14 +83,13 @@ public class NioTransport extends TcpTransport {
     @Override
     protected TcpNioServerSocketChannel bind(String name, InetSocketAddress address) throws IOException {
         TcpChannelFactory channelFactory = this.profileToChannelFactory.get(name);
-        AcceptingSelector selector = acceptors.get(++acceptorNumber % NioTransport.NIO_ACCEPTOR_COUNT.get(settings));
-        return channelFactory.openNioServerSocketChannel(address, selector);
+        return nioGroup.bindServerChannel(address, channelFactory);
     }
 
     @Override
     protected TcpNioSocketChannel initiateChannel(DiscoveryNode node, TimeValue connectTimeout, ActionListener connectListener)
         throws IOException {
-        TcpNioSocketChannel channel = clientChannelFactory.openNioChannel(node.getAddress().address(), clientSelectorSupplier.get());
+        TcpNioSocketChannel channel = nioGroup.openChannel(node.getAddress().address(), clientChannelFactory);
         channel.addConnectListener(connectListener);
         return channel;
     }
@@ -103,42 +98,19 @@ public class NioTransport extends TcpTransport {
     protected void doStart() {
         boolean success = false;
         try {
-            int workerCount = NioTransport.NIO_WORKER_COUNT.get(settings);
-            for (int i = 0; i < workerCount; ++i) {
-                SocketSelector selector = new SocketSelector(getSocketEventHandler());
-                socketSelectors.add(selector);
+            int acceptorCount = 0;
+            boolean useNetworkServer = NetworkService.NETWORK_SERVER.get(settings);
+            if (useNetworkServer) {
+                acceptorCount = NioTransport.NIO_ACCEPTOR_COUNT.get(settings);
             }
+            nioGroup = new NioGroup(logger, daemonThreadFactory(this.settings, TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX), acceptorCount,
+                AcceptorEventHandler::new, daemonThreadFactory(this.settings, TRANSPORT_WORKER_THREAD_NAME_PREFIX),
+                NioTransport.NIO_WORKER_COUNT.get(settings), this::getSocketEventHandler);
 
-            for (SocketSelector selector : socketSelectors) {
-                if (selector.isRunning() == false) {
-                    ThreadFactory threadFactory = daemonThreadFactory(this.settings, TRANSPORT_WORKER_THREAD_NAME_PREFIX);
-                    threadFactory.newThread(selector::runLoop).start();
-                    selector.isRunningFuture().actionGet();
-                }
-            }
-
-            Consumer clientContextSetter = getContextSetter("client-socket");
-            clientSelectorSupplier = new RoundRobinSelectorSupplier(socketSelectors);
             ProfileSettings clientProfileSettings = new ProfileSettings(settings, "default");
-            clientChannelFactory = new TcpChannelFactory(clientProfileSettings, clientContextSetter, getServerContextSetter());
-
-            if (NetworkService.NETWORK_SERVER.get(settings)) {
-                int acceptorCount = NioTransport.NIO_ACCEPTOR_COUNT.get(settings);
-                for (int i = 0; i < acceptorCount; ++i) {
-                    Supplier selectorSupplier = new RoundRobinSelectorSupplier(socketSelectors);
-                    AcceptorEventHandler eventHandler = new AcceptorEventHandler(logger, selectorSupplier);
-                    AcceptingSelector acceptor = new AcceptingSelector(eventHandler);
-                    acceptors.add(acceptor);
-                }
-
-                for (AcceptingSelector acceptor : acceptors) {
-                    if (acceptor.isRunning() == false) {
-                        ThreadFactory threadFactory = daemonThreadFactory(this.settings, TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX);
-                        threadFactory.newThread(acceptor::runLoop).start();
-                        acceptor.isRunningFuture().actionGet();
-                    }
-                }
+            clientChannelFactory = new TcpChannelFactory(clientProfileSettings, getContextSetter("client"), getServerContextSetter());
 
+            if (useNetworkServer) {
                 // loop through all profiles and start them up, special handling for default one
                 for (ProfileSettings profileSettings : profileSettings) {
                     String profileName = profileSettings.profileName;
@@ -162,14 +134,15 @@ public class NioTransport extends TcpTransport {
 
     @Override
     protected void stopInternal() {
-        NioShutdown nioShutdown = new NioShutdown(logger);
-        nioShutdown.orderlyShutdown(acceptors, socketSelectors);
-
+        try {
+            nioGroup.close();
+        } catch (Exception e) {
+            logger.warn("unexpected exception while stopping nio group", e);
+        }
         profileToChannelFactory.clear();
-        socketSelectors.clear();
     }
 
-    protected SocketEventHandler getSocketEventHandler() {
+    protected SocketEventHandler getSocketEventHandler(Logger logger) {
         return new SocketEventHandler(logger);
     }
 
@@ -189,8 +162,7 @@ public class NioTransport extends TcpTransport {
     }
 
     private void acceptChannel(NioSocketChannel channel) {
-        TcpNioSocketChannel tcpChannel = (TcpNioSocketChannel) channel;
-        serverAcceptedChannel(tcpChannel);
+        serverAcceptedChannel((TcpNioSocketChannel) channel);
 
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/RoundRobinSelectorSupplier.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/RoundRobinSupplier.java
similarity index 73%
rename from test/framework/src/main/java/org/elasticsearch/transport/nio/RoundRobinSelectorSupplier.java
rename to test/framework/src/main/java/org/elasticsearch/transport/nio/RoundRobinSupplier.java
index 108242b1e0e..395b955f7ab 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/RoundRobinSelectorSupplier.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/RoundRobinSupplier.java
@@ -19,22 +19,21 @@
 
 package org.elasticsearch.transport.nio;
 
-import java.util.ArrayList;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 
-public class RoundRobinSelectorSupplier implements Supplier {
+public class RoundRobinSupplier implements Supplier {
 
-    private final ArrayList selectors;
+    private final S[] selectors;
     private final int count;
     private AtomicInteger counter = new AtomicInteger(0);
 
-    public RoundRobinSelectorSupplier(ArrayList selectors) {
-        this.count = selectors.size();
+    public RoundRobinSupplier(S[] selectors) {
+        this.count = selectors.length;
         this.selectors = selectors;
     }
 
-    public SocketSelector get() {
-        return selectors.get(counter.getAndIncrement() % count);
+    public S get() {
+        return selectors[counter.getAndIncrement() % count];
     }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/AcceptorEventHandlerTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/AcceptorEventHandlerTests.java
index aedff1721f8..48a9e65f00d 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/AcceptorEventHandlerTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/AcceptorEventHandlerTests.java
@@ -57,7 +57,7 @@ public class AcceptorEventHandlerTests extends ESTestCase {
         acceptedChannelCallback = mock(Consumer.class);
         ArrayList selectors = new ArrayList<>();
         selectors.add(socketSelector);
-        handler = new AcceptorEventHandler(logger, new RoundRobinSelectorSupplier(selectors));
+        handler = new AcceptorEventHandler(logger, new RoundRobinSupplier<>(selectors.toArray(new SocketSelector[selectors.size()])));
 
         AcceptingSelector selector = mock(AcceptingSelector.class);
         channel = new DoNotRegisterServerChannel(mock(ServerSocketChannel.class), channelFactory, selector);
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/NioGroupTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/NioGroupTests.java
new file mode 100644
index 00000000000..f9b3cbb4e50
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/NioGroupTests.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport.nio;
+
+import org.elasticsearch.common.CheckedRunnable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.nio.channel.ChannelFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
+import static org.mockito.Mockito.mock;
+
+public class NioGroupTests extends ESTestCase {
+
+    private NioGroup nioGroup;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        nioGroup = new NioGroup(logger, daemonThreadFactory(Settings.EMPTY, "acceptor"), 1, AcceptorEventHandler::new,
+            daemonThreadFactory(Settings.EMPTY, "selector"), 1, SocketEventHandler::new);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        nioGroup.close();
+        super.tearDown();
+    }
+
+    public void testStartAndClose() throws IOException {
+        // ctor starts threads. So we are testing that close() stops the threads. Our thread linger checks
+        // will throw an exception is stop fails
+        nioGroup.close();
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testCannotOperateAfterClose() throws IOException {
+        nioGroup.close();
+
+        IllegalStateException ise = expectThrows(IllegalStateException.class,
+            () -> nioGroup.bindServerChannel(mock(InetSocketAddress.class), mock(ChannelFactory.class)));
+        assertEquals("NioGroup is closed.", ise.getMessage());
+        ise = expectThrows(IllegalStateException.class,
+            () -> nioGroup.openChannel(mock(InetSocketAddress.class), mock(ChannelFactory.class)));
+        assertEquals("NioGroup is closed.", ise.getMessage());
+    }
+
+    public void testCanCloseTwice() throws IOException {
+        nioGroup.close();
+        nioGroup.close();
+    }
+
+    public void testExceptionAtStartIsHandled() throws IOException {
+        RuntimeException ex = new RuntimeException();
+        CheckedRunnable ctor = () -> new NioGroup(logger, r -> {throw ex;}, 1,
+            AcceptorEventHandler::new, daemonThreadFactory(Settings.EMPTY, "selector"), 1, SocketEventHandler::new);
+        RuntimeException runtimeException = expectThrows(RuntimeException.class, ctor::run);
+        assertSame(ex, runtimeException);
+        // ctor starts threads. So we are testing that a failure to construct will stop threads. Our thread
+        // linger checks will throw an exception is stop fails
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java
index 1cff80dec79..1f17c3df541 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.transport.nio;
 
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -29,7 +30,6 @@ import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
-import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -78,7 +78,7 @@ public class SimpleNioTransportTests extends AbstractSimpleTransportTestCase {
             }
 
             @Override
-            protected SocketEventHandler getSocketEventHandler() {
+            protected SocketEventHandler getSocketEventHandler(Logger logger) {
                 return new TestingSocketEventHandler(logger);
             }
         };

From 541e142a6d1575eb55432541c26fe08ad7c5477a Mon Sep 17 00:00:00 2001
From: jaymode 
Date: Fri, 15 Dec 2017 10:41:37 -0700
Subject: [PATCH 284/297] Revert "Increase logging on qa:mixed-cluster tests"

This reverts commit e04e5ab037f6598521cd3707cb57a26a85f364d8 as we no longer need the increased
logging for the mixed cluster tests. This will reduce the size of logs for some build failures.
---
 qa/mixed-cluster/build.gradle | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle
index c5806b3546c..71b9a4993f6 100644
--- a/qa/mixed-cluster/build.gradle
+++ b/qa/mixed-cluster/build.gradle
@@ -45,9 +45,6 @@ for (Version version : versionCollection.versionsWireCompatibleWithCurrent) {
     numNodes = 4
     numBwcNodes = 2
     bwcVersion = version
-
-    setting 'logger.level', 'DEBUG'
-    setting 'logger.org.elasticsearch.discovery', 'TRACE'
   }
 
   Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {

From 717e2ddf4219426c9cf52b3f8531ad29ed32a74d Mon Sep 17 00:00:00 2001
From: Jay Modi 
Date: Fri, 15 Dec 2017 12:54:22 -0700
Subject: [PATCH 285/297] Do not use system properties when building the
 HttpAsyncClient (#27829)

This commit removes the usage of system properties for the HttpAsyncClient as this overrides some
defaults that we intentionally change. In order to set the default SSLContext to the system context
we set the SSLContext on the builder explicitly.

Closes #27827
---
 .../client/RestClientBuilder.java             | 31 ++++++++++++-------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java
index 38c9cdbe6e6..286ed7dd539 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java
@@ -28,7 +28,9 @@ import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
 import org.apache.http.nio.conn.SchemeIOSessionStrategy;
 
+import javax.net.ssl.SSLContext;
 import java.security.AccessController;
+import java.security.NoSuchAlgorithmException;
 import java.security.PrivilegedAction;
 import java.util.Objects;
 
@@ -200,20 +202,25 @@ public final class RestClientBuilder {
             requestConfigBuilder = requestConfigCallback.customizeRequestConfig(requestConfigBuilder);
         }
 
-        HttpAsyncClientBuilder httpClientBuilder = HttpAsyncClientBuilder.create().setDefaultRequestConfig(requestConfigBuilder.build())
+        try {
+            HttpAsyncClientBuilder httpClientBuilder = HttpAsyncClientBuilder.create().setDefaultRequestConfig(requestConfigBuilder.build())
                 //default settings for connection pooling may be too constraining
-                .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE).setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL).useSystemProperties();
-        if (httpClientConfigCallback != null) {
-            httpClientBuilder = httpClientConfigCallback.customizeHttpClient(httpClientBuilder);
-        }
-
-        final HttpAsyncClientBuilder finalBuilder = httpClientBuilder;
-        return AccessController.doPrivileged(new PrivilegedAction() {
-            @Override
-            public CloseableHttpAsyncClient run() {
-                return finalBuilder.build();
+                .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE).setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL)
+                .setSSLContext(SSLContext.getDefault());
+            if (httpClientConfigCallback != null) {
+                httpClientBuilder = httpClientConfigCallback.customizeHttpClient(httpClientBuilder);
             }
-        });
+
+            final HttpAsyncClientBuilder finalBuilder = httpClientBuilder;
+            return AccessController.doPrivileged(new PrivilegedAction() {
+                @Override
+                public CloseableHttpAsyncClient run() {
+                    return finalBuilder.build();
+                }
+            });
+        } catch (NoSuchAlgorithmException e) {
+            throw new IllegalStateException("could not create the default ssl context", e);
+        }
     }
 
     /**

From 43ff38c5dabcd862e260db28efd951d077cdf777 Mon Sep 17 00:00:00 2001
From: Tal Levy 
Date: Fri, 15 Dec 2017 13:47:26 -0800
Subject: [PATCH 286/297] update ingest-attachment to use Tika 1.17 and newer
 deps (#27824)

- this pr updates tika and its dependencies
- updates the SHAs
- updates the class excludes
---
 plugins/ingest-attachment/build.gradle        | 113 +++++++++-------
 .../apache-mime4j-core-0.7.2.jar.sha1         |   1 -
 .../apache-mime4j-core-0.8.1.jar.sha1         |   1 +
 .../licenses/apache-mime4j-dom-0.7.2.jar.sha1 |   1 -
 .../licenses/apache-mime4j-dom-0.8.1.jar.sha1 |   1 +
 .../licenses/commons-io-2.4.jar.sha1          |   1 -
 .../licenses/commons-io-2.5.jar.sha1          |   1 +
 .../licenses/fontbox-2.0.3.jar.sha1           |   1 -
 .../licenses/fontbox-2.0.8.jar.sha1           |   1 +
 .../licenses/jempbox-1.8.12.jar.sha1          |   1 -
 .../licenses/jempbox-1.8.13.jar.sha1          |   1 +
 .../licenses/pdfbox-2.0.3.jar.sha1            |   1 -
 .../licenses/pdfbox-2.0.8.jar.sha1            |   1 +
 .../licenses/poi-3.16.jar.sha1                |   1 -
 .../licenses/poi-3.17.jar.sha1                |   1 +
 .../licenses/poi-ooxml-3.16.jar.sha1          |   1 -
 .../licenses/poi-ooxml-3.17.jar.sha1          |   1 +
 .../licenses/poi-ooxml-schemas-3.16.jar.sha1  |   1 -
 .../licenses/poi-ooxml-schemas-3.17.jar.sha1  |   1 +
 .../licenses/poi-scratchpad-3.16.jar.sha1     |   1 -
 .../licenses/poi-scratchpad-3.17.jar.sha1     |   1 +
 .../licenses/tika-core-1.15.jar.sha1          |   1 -
 .../licenses/tika-core-1.17.jar.sha1          |   1 +
 .../licenses/tika-parsers-1.15.jar.sha1       |   1 -
 .../licenses/tika-parsers-1.17.jar.sha1       |   1 +
 .../attachment/AttachmentProcessor.java       | 123 +++++++++---------
 26 files changed, 144 insertions(+), 116 deletions(-)
 delete mode 100644 plugins/ingest-attachment/licenses/apache-mime4j-core-0.7.2.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.1.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/apache-mime4j-dom-0.7.2.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.1.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/commons-io-2.4.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/commons-io-2.5.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/fontbox-2.0.3.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/fontbox-2.0.8.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/jempbox-1.8.12.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/jempbox-1.8.13.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/pdfbox-2.0.3.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/pdfbox-2.0.8.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-3.16.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-3.17.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-3.16.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-3.17.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.16.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.17.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-scratchpad-3.16.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-scratchpad-3.17.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/tika-core-1.15.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/tika-core-1.17.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/tika-parsers-1.15.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/tika-parsers-1.17.jar.sha1

diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle
index b79501966e3..a57d8f880bc 100644
--- a/plugins/ingest-attachment/build.gradle
+++ b/plugins/ingest-attachment/build.gradle
@@ -23,11 +23,11 @@ esplugin {
 }
 
 versions << [
-  'tika': '1.15',
-  'pdfbox': '2.0.3',
+  'tika': '1.17',
+  'pdfbox': '2.0.8',
   'bouncycastle': '1.55',
-  'poi': '3.16',
-  'mime4j': '0.7.2'
+  'poi': '3.17',
+  'mime4j': '0.8.1'
 ]
 
 dependencies {
@@ -35,7 +35,7 @@ dependencies {
   compile "org.apache.tika:tika-core:${versions.tika}"
   compile "org.apache.tika:tika-parsers:${versions.tika}"
   compile 'org.tukaani:xz:1.6'
-  compile 'commons-io:commons-io:2.4'
+  compile 'commons-io:commons-io:2.5'
   compile "org.slf4j:slf4j-api:${versions.slf4j}"
 
   // character set detection
@@ -47,7 +47,7 @@ dependencies {
   // Adobe PDF
   compile "org.apache.pdfbox:pdfbox:${versions.pdfbox}"
   compile "org.apache.pdfbox:fontbox:${versions.pdfbox}"
-  compile "org.apache.pdfbox:jempbox:1.8.12"
+  compile "org.apache.pdfbox:jempbox:1.8.13"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "org.bouncycastle:bcmail-jdk15on:${versions.bouncycastle}"
   compile "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
@@ -546,6 +546,7 @@ thirdPartyAudit.excludes = [
   'org.apache.http.client.utils.URIBuilder',
   'org.apache.http.entity.ByteArrayEntity',
   'org.apache.http.impl.client.DefaultHttpClient',
+  'org.apache.jcp.xml.dsig.internal.dom.ApacheNodeSetData',
   'org.apache.jcp.xml.dsig.internal.dom.DOMDigestMethod',
   'org.apache.jcp.xml.dsig.internal.dom.DOMKeyInfo',
   'org.apache.jcp.xml.dsig.internal.dom.DOMReference',
@@ -588,6 +589,7 @@ thirdPartyAudit.excludes = [
   'org.apache.uima.util.XmlCasSerializer',
   'org.apache.xml.security.Init',
   'org.apache.xml.security.c14n.Canonicalizer',
+  'org.apache.xml.security.signature.XMLSignatureInput',
   'org.apache.xml.security.utils.Base64',
   'org.brotli.dec.BrotliInputStream',
   'org.etsi.uri.x01903.v13.AnyType',
@@ -635,11 +637,9 @@ thirdPartyAudit.excludes = [
   'org.etsi.uri.x01903.v14.ValidationDataType',
   'org.json.JSONArray',
   'org.json.JSONObject',
-  'org.json.XML',
   'org.json.simple.JSONArray',
   'org.json.simple.JSONObject',
   'org.json.simple.parser.JSONParser',
-  'org.junit.Assert',
   'org.junit.Test',
   'org.junit.internal.TextListener',
   'org.junit.runner.JUnitCore',
@@ -690,7 +690,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTDLbls',
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTDPt',
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTDTable',
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDateAx',
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTDispBlanksAs',
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTDispUnits',
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTDoughnutChart',
@@ -720,6 +719,7 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTSurface3DChart',
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTSurfaceChart',
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTTextLanguageID',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTTimeUnit',
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTTrendline',
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTUpDownBars',
   'org.openxmlformats.schemas.drawingml.x2006.chart.CTView3D',
@@ -828,8 +828,8 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.STOnOffStyleType$Enum',
   'org.openxmlformats.schemas.drawingml.x2006.main.STPanose',
   'org.openxmlformats.schemas.drawingml.x2006.main.STPathFillMode',
-  'org.openxmlformats.schemas.drawingml.x2006.main.STPresetPatternVal',
   'org.openxmlformats.schemas.drawingml.x2006.main.STPresetPatternVal$Enum',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STPresetPatternVal',
   'org.openxmlformats.schemas.drawingml.x2006.main.STRectAlignment',
   'org.openxmlformats.schemas.drawingml.x2006.main.STTextColumnCount',
   'org.openxmlformats.schemas.drawingml.x2006.main.STTextNonNegativePoint',
@@ -937,34 +937,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1SatOffList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1ShadeList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1TintList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaModList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaOffList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueModList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueOffList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1CompList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GammaList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GrayList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenModList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenOffList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueModList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueOffList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1InvGammaList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1InvList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumModList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumOffList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedModList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedOffList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatModList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatOffList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1ShadeList',
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1TintList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaModList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaOffList',
@@ -993,6 +965,34 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1SatOffList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1ShadeList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1TintList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1AlphaOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1BlueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1CompList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GrayList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1GreenOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1HueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1InvGammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1InvList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1LumOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1RedOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1SatOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1ShadeList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTScRgbColorImpl$1TintList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1AlphaList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1AlphaModList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1AlphaOffList',
@@ -1058,7 +1058,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextParagraphImpl$1FldList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextParagraphImpl$1RList',
   'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextTabStopListImpl$1TabList',
-  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTAbsoluteAnchor',
   'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTDrawingImpl$1AbsoluteAnchorList',
   'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTDrawingImpl$1OneCellAnchorList',
   'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTDrawingImpl$1TwoCellAnchorList',
@@ -1183,7 +1182,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTChartsheetViews',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTColHierarchiesUsage',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTColItems',
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTColors',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTConditionalFormats',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTConsolidation',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTControls',
@@ -1208,6 +1206,7 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFormats',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFunctionGroups',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTGradientFill',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMRUColors',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMeasureDimensionMaps',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMeasureGroups',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMissing',
@@ -1231,7 +1230,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSortState',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTString',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTableFormula',
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTableStyles',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTupleCache',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTWebPublishItems',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTWebPublishObjects',
@@ -1333,14 +1331,14 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1NList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1SList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetDataImpl$1RowList',
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetsImpl$1SheetList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetViewImpl$1PivotSelectionList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetViewImpl$1SelectionList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetViewsImpl$1SheetViewList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetsImpl$1SheetList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSingleXmlCellsImpl$1SingleXmlCellList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSstImpl$1SiList',
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTTableColumnsImpl$1TableColumnList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTTablePartsImpl$1TablePartList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTTableStylesImpl$1TableStyleList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTWorkbookImpl$1FileRecoveryPrList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTWorksheetImpl$1ColsList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTWorksheetImpl$1ConditionalFormattingList',
@@ -1400,7 +1398,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTProof',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTRPrChange',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTReadingModeInkLockDown',
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTRuby',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSaveThroughXslt',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtComboBox',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtDate',
@@ -1441,6 +1438,7 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.STPTabLeader',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.STPTabRelativeTo',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.STProofErr',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STRubyAlign',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.STShortHexNumber',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.STThemeColor',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.STUcharHexNumber',
@@ -1708,6 +1706,32 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1ProofErrList',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1SdtList',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1TcList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRubyContentImpl$1RList',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1AccList',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BarList',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BookmarkEndList',
@@ -2054,7 +2078,6 @@ thirdPartyAudit.excludes = [
   'org.sqlite.SQLiteConfig',
   'org.w3.x2000.x09.xmldsig.KeyInfoType',
   'org.w3.x2000.x09.xmldsig.SignatureMethodType',
-  'org.w3.x2000.x09.xmldsig.SignatureValueType',
   'org.w3.x2000.x09.xmldsig.TransformsType',
   'org.w3.x2000.x09.xmldsig.impl.SignatureTypeImpl$1ObjectList',
   'org.w3.x2000.x09.xmldsig.impl.SignedInfoTypeImpl$1ReferenceList',
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-core-0.7.2.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-core-0.7.2.jar.sha1
deleted file mode 100644
index 8210fc7fc16..00000000000
--- a/plugins/ingest-attachment/licenses/apache-mime4j-core-0.7.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a81264fe0265ebe8fd1d8128aad06dc320de6eef
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.1.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.1.jar.sha1
new file mode 100644
index 00000000000..6ae3e58d22b
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.1.jar.sha1
@@ -0,0 +1 @@
+c62dfe18a3b827a2c626ade0ffba44562ddf3f61
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.7.2.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.7.2.jar.sha1
deleted file mode 100644
index 3ede85a0191..00000000000
--- a/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.7.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1c289aa264548a0a1f1b43685a9cb2ab23f67287
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.1.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.1.jar.sha1
new file mode 100644
index 00000000000..408dfe12ef2
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.1.jar.sha1
@@ -0,0 +1 @@
+f2d653c617004193f3350330d907f77b60c88c56
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-io-2.4.jar.sha1 b/plugins/ingest-attachment/licenses/commons-io-2.4.jar.sha1
deleted file mode 100644
index 688318c938c..00000000000
--- a/plugins/ingest-attachment/licenses/commons-io-2.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b1b6ea3b7e4aa4f492509a4952029cd8e48019ad
diff --git a/plugins/ingest-attachment/licenses/commons-io-2.5.jar.sha1 b/plugins/ingest-attachment/licenses/commons-io-2.5.jar.sha1
new file mode 100644
index 00000000000..b7f1d93e897
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/commons-io-2.5.jar.sha1
@@ -0,0 +1 @@
+2852e6e05fbb95076fc091f6d1780f1f8fe35e0f
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/fontbox-2.0.3.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-2.0.3.jar.sha1
deleted file mode 100644
index e3ff3d39459..00000000000
--- a/plugins/ingest-attachment/licenses/fontbox-2.0.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-448ee588d0136121cf5c4dd397384cccb9db1ad7
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/fontbox-2.0.8.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-2.0.8.jar.sha1
new file mode 100644
index 00000000000..f8abddbc755
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/fontbox-2.0.8.jar.sha1
@@ -0,0 +1 @@
+52f852fcfc7481d45efdffd224eb78b85981b17b
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/jempbox-1.8.12.jar.sha1 b/plugins/ingest-attachment/licenses/jempbox-1.8.12.jar.sha1
deleted file mode 100644
index 0e3dcf4573b..00000000000
--- a/plugins/ingest-attachment/licenses/jempbox-1.8.12.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-426450c573c19f6f2c751a7a52c11931b712c9f6
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/jempbox-1.8.13.jar.sha1 b/plugins/ingest-attachment/licenses/jempbox-1.8.13.jar.sha1
new file mode 100644
index 00000000000..2593719dfb3
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/jempbox-1.8.13.jar.sha1
@@ -0,0 +1 @@
+a874cef0ed0e2a8c4cc5ed52c23ba3e6d78eca4e
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/pdfbox-2.0.3.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-2.0.3.jar.sha1
deleted file mode 100644
index 807e2482ac2..00000000000
--- a/plugins/ingest-attachment/licenses/pdfbox-2.0.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-be7b09de93f7c7795c57f4fbf14db60ab93806b4
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/pdfbox-2.0.8.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-2.0.8.jar.sha1
new file mode 100644
index 00000000000..1c346871e21
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/pdfbox-2.0.8.jar.sha1
@@ -0,0 +1 @@
+17bdf273d66f3afe41eedb9d3ab6a7b819c44a0c
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-3.16.jar.sha1 b/plugins/ingest-attachment/licenses/poi-3.16.jar.sha1
deleted file mode 100644
index 75cbf233847..00000000000
--- a/plugins/ingest-attachment/licenses/poi-3.16.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ad21c123ee5d6b5b2a8f0d4ed23b3ffe6759a889
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-3.17.jar.sha1 b/plugins/ingest-attachment/licenses/poi-3.17.jar.sha1
new file mode 100644
index 00000000000..bd472c0bec7
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-3.17.jar.sha1
@@ -0,0 +1 @@
+0ae92292a2043888b40d418da97dc0b669fde326
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-3.16.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-3.16.jar.sha1
deleted file mode 100644
index c2283c7fa1a..00000000000
--- a/plugins/ingest-attachment/licenses/poi-ooxml-3.16.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-76e20fe22404cc4da55ddfdaaaadee32bbfa3bdd
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-3.17.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-3.17.jar.sha1
new file mode 100644
index 00000000000..37c5e068814
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-ooxml-3.17.jar.sha1
@@ -0,0 +1 @@
+07d8c44407178b73246462842bf1e206e99c8e0a
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.16.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.16.jar.sha1
deleted file mode 100644
index 8ddbb300169..00000000000
--- a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.16.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9828a49307fc6bebfd42185b677d88b6e4994c63
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.17.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.17.jar.sha1
new file mode 100644
index 00000000000..744e323e5d7
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.17.jar.sha1
@@ -0,0 +1 @@
+890114bfa82f5b6380ea0e9b0bf49b0af797b414
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-scratchpad-3.16.jar.sha1 b/plugins/ingest-attachment/licenses/poi-scratchpad-3.16.jar.sha1
deleted file mode 100644
index 8dc53c0bfbc..00000000000
--- a/plugins/ingest-attachment/licenses/poi-scratchpad-3.16.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-69d6dda524e38a491b362d0f94ef74a514faf70a
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-scratchpad-3.17.jar.sha1 b/plugins/ingest-attachment/licenses/poi-scratchpad-3.17.jar.sha1
new file mode 100644
index 00000000000..16686b3e89b
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-scratchpad-3.17.jar.sha1
@@ -0,0 +1 @@
+85d86a0e26c7f5c0db4ee63e8c7728e51c5d64ce
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-core-1.15.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-1.15.jar.sha1
deleted file mode 100644
index cc764669b5b..00000000000
--- a/plugins/ingest-attachment/licenses/tika-core-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-17850c2224e4e3867e588060dc8ce6ba3bcfab2a
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-core-1.17.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-1.17.jar.sha1
new file mode 100644
index 00000000000..571314b3378
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-core-1.17.jar.sha1
@@ -0,0 +1 @@
+b450102c2aee98107474d2f92661d947b9cef183
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parsers-1.15.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-1.15.jar.sha1
deleted file mode 100644
index cada2d9f3ac..00000000000
--- a/plugins/ingest-attachment/licenses/tika-parsers-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-aa07c2cda051709e5fe70fd6e244386fc93b0a1e
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parsers-1.17.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-1.17.jar.sha1
new file mode 100644
index 00000000000..c4487e4970f
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parsers-1.17.jar.sha1
@@ -0,0 +1 @@
+4277c54fcaed542fbc8a0001fdb4c23baccc0132
\ No newline at end of file
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
index f7f474711be..b23c627290e 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.ingest.attachment;
 
+import org.apache.tika.exception.ZeroByteFileException;
 import org.apache.tika.language.LanguageIdentifier;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
@@ -81,70 +82,74 @@ public final class AttachmentProcessor extends AbstractProcessor {
             throw new IllegalArgumentException("field [" + field + "] is null, cannot parse.");
         }
 
+        Metadata metadata = new Metadata();
+        String parsedContent = "";
         try {
-            Metadata metadata = new Metadata();
-            String parsedContent = TikaImpl.parse(input, metadata, indexedChars);
-
-            if (properties.contains(Property.CONTENT) && Strings.hasLength(parsedContent)) {
-                // somehow tika seems to append a newline at the end automatically, lets remove that again
-                additionalFields.put(Property.CONTENT.toLowerCase(), parsedContent.trim());
-            }
-
-            if (properties.contains(Property.LANGUAGE) && Strings.hasLength(parsedContent)) {
-                LanguageIdentifier identifier = new LanguageIdentifier(parsedContent);
-                String language = identifier.getLanguage();
-                additionalFields.put(Property.LANGUAGE.toLowerCase(), language);
-            }
-
-            if (properties.contains(Property.DATE)) {
-                String createdDate = metadata.get(TikaCoreProperties.CREATED);
-                if (createdDate != null) {
-                    additionalFields.put(Property.DATE.toLowerCase(), createdDate);
-                }
-            }
-
-            if (properties.contains(Property.TITLE)) {
-                String title = metadata.get(TikaCoreProperties.TITLE);
-                if (Strings.hasLength(title)) {
-                    additionalFields.put(Property.TITLE.toLowerCase(), title);
-                }
-            }
-
-            if (properties.contains(Property.AUTHOR)) {
-                String author = metadata.get("Author");
-                if (Strings.hasLength(author)) {
-                    additionalFields.put(Property.AUTHOR.toLowerCase(), author);
-                }
-            }
-
-            if (properties.contains(Property.KEYWORDS)) {
-                String keywords = metadata.get("Keywords");
-                if (Strings.hasLength(keywords)) {
-                    additionalFields.put(Property.KEYWORDS.toLowerCase(), keywords);
-                }
-            }
-
-            if (properties.contains(Property.CONTENT_TYPE)) {
-                String contentType = metadata.get(Metadata.CONTENT_TYPE);
-                if (Strings.hasLength(contentType)) {
-                    additionalFields.put(Property.CONTENT_TYPE.toLowerCase(), contentType);
-                }
-            }
-
-            if (properties.contains(Property.CONTENT_LENGTH)) {
-                String contentLength = metadata.get(Metadata.CONTENT_LENGTH);
-                long length;
-                if (Strings.hasLength(contentLength)) {
-                    length = Long.parseLong(contentLength);
-                } else {
-                    length = parsedContent.length();
-                }
-                additionalFields.put(Property.CONTENT_LENGTH.toLowerCase(), length);
-            }
+            parsedContent = TikaImpl.parse(input, metadata, indexedChars);
+        } catch (ZeroByteFileException e) {
+            // tika 1.17 throws an exception when the InputStream has 0 bytes.
+            // previously, it did not mind. This is here to preserve that behavior.
         } catch (Exception e) {
             throw new ElasticsearchParseException("Error parsing document in field [{}]", e, field);
         }
 
+        if (properties.contains(Property.CONTENT) && Strings.hasLength(parsedContent)) {
+            // somehow tika seems to append a newline at the end automatically, lets remove that again
+            additionalFields.put(Property.CONTENT.toLowerCase(), parsedContent.trim());
+        }
+
+        if (properties.contains(Property.LANGUAGE) && Strings.hasLength(parsedContent)) {
+            LanguageIdentifier identifier = new LanguageIdentifier(parsedContent);
+            String language = identifier.getLanguage();
+            additionalFields.put(Property.LANGUAGE.toLowerCase(), language);
+        }
+
+        if (properties.contains(Property.DATE)) {
+            String createdDate = metadata.get(TikaCoreProperties.CREATED);
+            if (createdDate != null) {
+                additionalFields.put(Property.DATE.toLowerCase(), createdDate);
+            }
+        }
+
+        if (properties.contains(Property.TITLE)) {
+            String title = metadata.get(TikaCoreProperties.TITLE);
+            if (Strings.hasLength(title)) {
+                additionalFields.put(Property.TITLE.toLowerCase(), title);
+            }
+        }
+
+        if (properties.contains(Property.AUTHOR)) {
+            String author = metadata.get("Author");
+            if (Strings.hasLength(author)) {
+                additionalFields.put(Property.AUTHOR.toLowerCase(), author);
+            }
+        }
+
+        if (properties.contains(Property.KEYWORDS)) {
+            String keywords = metadata.get("Keywords");
+            if (Strings.hasLength(keywords)) {
+                additionalFields.put(Property.KEYWORDS.toLowerCase(), keywords);
+            }
+        }
+
+        if (properties.contains(Property.CONTENT_TYPE)) {
+            String contentType = metadata.get(Metadata.CONTENT_TYPE);
+            if (Strings.hasLength(contentType)) {
+                additionalFields.put(Property.CONTENT_TYPE.toLowerCase(), contentType);
+            }
+        }
+
+        if (properties.contains(Property.CONTENT_LENGTH)) {
+            String contentLength = metadata.get(Metadata.CONTENT_LENGTH);
+            long length;
+            if (Strings.hasLength(contentLength)) {
+                length = Long.parseLong(contentLength);
+            } else {
+                length = parsedContent.length();
+            }
+            additionalFields.put(Property.CONTENT_LENGTH.toLowerCase(), length);
+        }
+
         ingestDocument.setFieldValue(targetField, additionalFields);
     }
 

From 4f62b51c87032e72e1ec4a1c36b9884ade8852fb Mon Sep 17 00:00:00 2001
From: Nhat Nguyen 
Date: Sat, 16 Dec 2017 11:03:31 -0500
Subject: [PATCH 287/297] Use lastSyncedGlobalCheckpoint in deletion policy
 (#27826)

Today we use the in-memory global checkpoint from SequenceNumbersService
to clean up unneeded commit points, however the latest global checkpoint
may haven't fsynced to the disk yet. If the translog checkpoint fsync
failed and we already use a higher global checkpoint to clean up commit
points, then we may have removed a safe commit which we try to keep for
recovery.

This commit updates the deletion policy using lastSyncedGlobalCheckpoint
from Translog rather the in memory global checkpoint.

Relates #27606
---
 .../index/engine/InternalEngine.java          | 23 ++++++++--------
 .../index/engine/InternalEngineTests.java     | 26 +++++++++++++------
 2 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 1a6dba8eb17..5bf6431296c 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -182,8 +182,11 @@ public class InternalEngine extends Engine {
                 final SeqNoStats seqNoStats = loadSeqNoStats(openMode);
                 logger.trace("recovered [{}]", seqNoStats);
                 this.seqNoService = seqNoServiceSupplier.apply(engineConfig, seqNoStats);
+                translog = openTranslog(engineConfig, translogDeletionPolicy, seqNoService::getGlobalCheckpoint);
+                assert translog.getGeneration() != null;
+                this.translog = translog;
                 this.snapshotDeletionPolicy = new SnapshotDeletionPolicy(
-                    new CombinedDeletionPolicy(openMode, translogDeletionPolicy, seqNoService::getGlobalCheckpoint)
+                    new CombinedDeletionPolicy(openMode, translogDeletionPolicy, translog::getLastSyncedGlobalCheckpoint)
                 );
                 writer = createWriter(openMode == EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG);
                 updateMaxUnsafeAutoIdTimestampFromWriter(writer);
@@ -195,9 +198,6 @@ public class InternalEngine extends Engine {
                 historyUUID = loadOrGenerateHistoryUUID(writer, engineConfig.getForceNewHistoryUUID());
                 Objects.requireNonNull(historyUUID, "history uuid should not be null");
                 indexWriter = writer;
-                translog = openTranslog(engineConfig, writer, translogDeletionPolicy, () -> seqNoService.getGlobalCheckpoint());
-                assert translog.getGeneration() != null;
-                this.translog = translog;
                 updateWriterOnOpen();
             } catch (IOException | TranslogCorruptedException e) {
                 throw new EngineCreationFailureException(shardId, "failed to create engine", e);
@@ -437,12 +437,12 @@ public class InternalEngine extends Engine {
         translog.trimUnreferencedReaders();
     }
 
-    private Translog openTranslog(EngineConfig engineConfig, IndexWriter writer, TranslogDeletionPolicy translogDeletionPolicy, LongSupplier globalCheckpointSupplier) throws IOException {
+    private Translog openTranslog(EngineConfig engineConfig, TranslogDeletionPolicy translogDeletionPolicy, LongSupplier globalCheckpointSupplier) throws IOException {
         assert openMode != null;
         final TranslogConfig translogConfig = engineConfig.getTranslogConfig();
         String translogUUID = null;
         if (openMode == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG) {
-            translogUUID = loadTranslogUUIDFromCommit(writer);
+            translogUUID = loadTranslogUUIDFromLastCommit();
             // We expect that this shard already exists, so it must already have an existing translog else something is badly wrong!
             if (translogUUID == null) {
                 throw new IndexFormatTooOldException("translog", "translog has no generation nor a UUID - this might be an index from a previous version consider upgrading to N-1 first");
@@ -492,14 +492,13 @@ public class InternalEngine extends Engine {
     }
 
     /**
-     * Reads the current stored translog ID from the IW commit data. If the id is not found, recommits the current
-     * translog id into lucene and returns null.
+     * Reads the current stored translog ID from the last commit data.
      */
     @Nullable
-    private String loadTranslogUUIDFromCommit(IndexWriter writer) throws IOException {
-        // commit on a just opened writer will commit even if there are no changes done to it
-        // we rely on that for the commit data translog id key
-        final Map commitUserData = commitDataAsMap(writer);
+    private String loadTranslogUUIDFromLastCommit() throws IOException {
+        assert openMode == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG :
+            "Only reuse existing translogUUID with OPEN_INDEX_AND_TRANSLOG; openMode = [" + openMode + "]";
+        final Map commitUserData = store.readLastCommittedSegmentsInfo().getUserData();
         if (commitUserData.containsKey(Translog.TRANSLOG_UUID_KEY)) {
             if (commitUserData.containsKey(Translog.TRANSLOG_GENERATION_KEY) == false) {
                 throw new IllegalStateException("commit doesn't contain translog generation id");
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 7e73228ecfd..03e4434f462 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -168,7 +168,6 @@ import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasKey;
-import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
@@ -4263,29 +4262,40 @@ public class InternalEngineTests extends EngineTestCase {
                 .put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), randomFrom("-1", "512b", "1gb")));
         indexSettings.updateIndexMetaData(builder.build());
 
+        final Path translogPath = createTempDir();
         store = createStore();
-        try (InternalEngine engine
-                 = createEngine(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null, seqNoServiceSupplier)) {
+        final EngineConfig engineConfig = config(indexSettings, store, translogPath, NoMergePolicy.INSTANCE, null);
+        try (Engine engine = new InternalEngine(engineConfig, seqNoServiceSupplier) {
+            @Override
+            protected void commitIndexWriter(IndexWriter writer, Translog translog, String syncId) throws IOException {
+                // Advance the global checkpoint during the flush to create a lag between a persisted global checkpoint in the translog
+                // (this value is visible to the deletion policy) and an in memory global checkpoint in the SequenceNumbersService.
+                if (rarely()) {
+                    globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), seqNoService().getLocalCheckpoint()));
+                }
+                super.commitIndexWriter(writer, translog, syncId);
+            }
+        }){
             int numDocs = scaledRandomIntBetween(10, 100);
             for (int docId = 0; docId < numDocs; docId++) {
                 ParseContext.Document document = testDocumentWithTextField();
                 document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
                 engine.index(indexForDoc(testParsedDocument(Integer.toString(docId), null, document, B_1, null)));
                 if (frequently()) {
-                    globalCheckpoint.set(randomIntBetween(
-                        Math.toIntExact(engine.seqNoService().getGlobalCheckpoint()),
-                        Math.toIntExact(engine.seqNoService().getLocalCheckpoint())));
+                    globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.seqNoService().getLocalCheckpoint()));
+                    engine.getTranslog().sync();
                 }
                 if (frequently()) {
+                    final long lastSyncedGlobalCheckpoint = Translog.readGlobalCheckpoint(translogPath);
                     engine.flush(randomBoolean(), true);
                     final List commits = DirectoryReader.listCommits(store.directory());
                     // Keep only one safe commit as the oldest commit.
                     final IndexCommit safeCommit = commits.get(0);
                     assertThat(Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)),
-                        lessThanOrEqualTo(globalCheckpoint.get()));
+                        lessThanOrEqualTo(lastSyncedGlobalCheckpoint));
                     for (int i = 1; i < commits.size(); i++) {
                         assertThat(Long.parseLong(commits.get(i).getUserData().get(SequenceNumbers.MAX_SEQ_NO)),
-                            greaterThan(globalCheckpoint.get()));
+                            greaterThan(lastSyncedGlobalCheckpoint));
                     }
                     // Make sure we keep all translog operations after the local checkpoint of the safe commit.
                     long localCheckpointFromSafeCommit = Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));

From fade828c5054fd90e59b159fe7e189e318404e11 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Sun, 17 Dec 2017 11:51:18 -0500
Subject: [PATCH 288/297] Fix publication of elasticsearch-cli to Maven

This commit addresses the publication of the elasticsearch-cli to
Maven. For now for simplicity we publish this to Maven so that it is
available as a transitive dependency for any artifacts that depend on
the core elasticsearch artifact. It is possible that in the future we
can simply exclude this dependency but for now this is the safest and
simplest approach that can happen in a patch release.

Relates #27853
---
 core/cli/build.gradle | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/core/cli/build.gradle b/core/cli/build.gradle
index fc93523f6b7..6f0e2e55eec 100644
--- a/core/cli/build.gradle
+++ b/core/cli/build.gradle
@@ -20,6 +20,17 @@
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 
 apply plugin: 'elasticsearch.build'
+apply plugin: 'nebula.optional-base'
+apply plugin: 'nebula.maven-base-publish'
+apply plugin: 'nebula.maven-scm'
+
+publishing {
+    publications {
+        nebula {
+            artifactId 'elasticsearch-cli'
+        }
+    }
+}
 
 archivesBaseName = 'elasticsearch-cli'
 

From 75c0cd06720bddb1afd76c3b37296d5233d2ce20 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Sun, 17 Dec 2017 14:27:10 -0500
Subject: [PATCH 289/297] Move range field mapper back to core

This commit moves the range field mapper back to core so that we can
remove the compile-time dependency of percolator on mapper-extras which
compilcates dependency management for the percolator client JAR, and
modules should not be intertwined like this anyway.

Relates #27854
---
 .../lucene/queries/BinaryDocValuesRangeQuery.java    |  1 +
 .../elasticsearch/index/mapper/BinaryRangeUtil.java  |  2 +-
 .../elasticsearch/index/mapper/RangeFieldMapper.java |  1 +
 .../org/elasticsearch/indices/IndicesModule.java     |  4 ++++
 .../BaseRandomBinaryDocValuesRangeQueryTestCase.java |  0
 .../queries/BinaryDocValuesRangeQueryTests.java      |  0
 .../DoubleRandomBinaryDocValuesRangeQueryTests.java  |  0
 .../FloatRandomBinaryDocValuesRangeQueryTests.java   |  0
 ...tAddressRandomBinaryDocValuesRangeQueryTests.java |  0
 .../IntegerRandomBinaryDocValuesRangeQueryTests.java |  0
 .../LongRandomBinaryDocValuesRangeQueryTests.java    |  0
 .../index/mapper/BinaryRangeUtilTests.java           |  0
 .../index/mapper/IpRangeFieldMapperTests.java        | 12 +++---------
 .../index/mapper/RangeFieldMapperTests.java          |  3 ++-
 .../RangeFieldQueryStringQueryBuilderTests.java      | 11 -----------
 .../index/mapper/RangeFieldTypeTests.java            |  2 +-
 .../index/mapper/MapperExtrasPlugin.java             |  3 ---
 modules/percolator/build.gradle                      |  2 --
 .../resources/rest-api-spec/test/range/10_basic.yml  |  0
 19 files changed, 13 insertions(+), 28 deletions(-)
 rename {modules/mapper-extras => core}/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java (99%)
 rename {modules/mapper-extras => core}/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java (99%)
 rename {modules/mapper-extras => core}/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java (99%)
 rename {modules/mapper-extras => core}/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java (100%)
 rename {modules/mapper-extras => core}/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java (100%)
 rename {modules/mapper-extras => core}/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java (100%)
 rename {modules/mapper-extras => core}/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java (100%)
 rename {modules/mapper-extras => core}/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java (100%)
 rename {modules/mapper-extras => core}/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java (100%)
 rename {modules/mapper-extras => core}/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java (100%)
 rename {modules/mapper-extras => core}/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java (100%)
 rename {modules/mapper-extras => core}/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java (94%)
 rename {modules/mapper-extras => core}/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java (99%)
 rename {modules/mapper-extras => core}/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java (95%)
 rename {modules/mapper-extras => core}/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java (100%)
 rename {modules/mapper-extras/src/test => rest-api-spec/src/main}/resources/rest-api-spec/test/range/10_basic.yml (100%)

diff --git a/modules/mapper-extras/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/core/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java
similarity index 99%
rename from modules/mapper-extras/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java
rename to core/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java
index b5caf997495..3cc16ce9320 100644
--- a/modules/mapper-extras/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java
+++ b/core/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.apache.lucene.queries;
 
 import org.apache.lucene.index.BinaryDocValues;
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/core/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java
similarity index 99%
rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java
rename to core/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java
index 384ab24a73b..b58c6deba8c 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java
@@ -16,9 +16,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.HalfFloatPoint;
 import org.apache.lucene.store.ByteArrayDataOutput;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
similarity index 99%
rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
rename to core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
index 46d553a4729..0f0b064e523 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.DoubleRange;
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
index f1e52409943..4e1456548ab 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -43,6 +43,7 @@ import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.mapper.RangeFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
@@ -98,6 +99,9 @@ public class IndicesModule extends AbstractModule {
         for (NumberFieldMapper.NumberType type : NumberFieldMapper.NumberType.values()) {
             mappers.put(type.typeName(), new NumberFieldMapper.TypeParser(type));
         }
+        for (RangeFieldMapper.RangeType type : RangeFieldMapper.RangeType.values()) {
+            mappers.put(type.typeName(), new RangeFieldMapper.TypeParser(type));
+        }
         mappers.put(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser());
         mappers.put(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser());
         mappers.put(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser());
diff --git a/modules/mapper-extras/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java b/core/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java
similarity index 100%
rename from modules/mapper-extras/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java
rename to core/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java
diff --git a/modules/mapper-extras/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java
similarity index 100%
rename from modules/mapper-extras/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java
rename to core/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java
diff --git a/modules/mapper-extras/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java
similarity index 100%
rename from modules/mapper-extras/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java
rename to core/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java
diff --git a/modules/mapper-extras/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java
similarity index 100%
rename from modules/mapper-extras/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java
rename to core/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java
diff --git a/modules/mapper-extras/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java
similarity index 100%
rename from modules/mapper-extras/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java
rename to core/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java
diff --git a/modules/mapper-extras/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java
similarity index 100%
rename from modules/mapper-extras/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java
rename to core/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java
diff --git a/modules/mapper-extras/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java
similarity index 100%
rename from modules/mapper-extras/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java
rename to core/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/core/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java
similarity index 100%
rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java
rename to core/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
similarity index 94%
rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
rename to core/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
index 63ebe7a6cb3..829c05701ff 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
@@ -18,9 +18,6 @@
  */
 package org.elasticsearch.index.mapper;
 
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -29,10 +26,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.junit.Before;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import static org.hamcrest.Matchers.containsString;
 
 public class IpRangeFieldMapperTests extends ESSingleNodeTestCase {
@@ -40,11 +39,6 @@ public class IpRangeFieldMapperTests extends ESSingleNodeTestCase {
     private IndexService indexService;
     private DocumentMapperParser parser;
 
-    @Override
-    protected Collection> getPlugins() {
-        return pluginList(MapperExtrasPlugin.class);
-    }
-
     @Before
     public void setup() {
         indexService = createIndex("test");
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
similarity index 99%
rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
rename to core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
index 0742aeadcb5..eea71525c70 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.InetAddressPoint;
@@ -48,7 +49,7 @@ public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase {
 
     @Override
     protected Collection> getPlugins() {
-        return pluginList(InternalSettingsPlugin.class, MapperExtrasPlugin.class);
+        return pluginList(InternalSettingsPlugin.class);
     }
 
     private static String FROM_DATE = "2016-10-31";
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java
similarity index 95%
rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java
rename to core/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java
index 654da9ae67b..72195fbd954 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.DoubleRange;
 import org.apache.lucene.document.FloatRange;
-import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.document.InetAddressRange;
 import org.apache.lucene.document.IntRange;
 import org.apache.lucene.document.LongRange;
@@ -29,22 +28,17 @@ import org.apache.lucene.queries.BinaryDocValuesRangeQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.joda.DateMathParser;
-import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryStringQueryBuilder;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.net.InetAddress;
-import java.util.Collection;
-import java.util.Collections;
 
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.core.IsInstanceOf.instanceOf;
@@ -58,11 +52,6 @@ public class RangeFieldQueryStringQueryBuilderTests extends AbstractQueryTestCas
     private static final String DATE_RANGE_FIELD_NAME = "mapped_date_range";
     private static final String IP_RANGE_FIELD_NAME = "mapped_ip_range";
 
-    @Override
-    protected Collection> getPlugins() {
-        return Collections.singleton(MapperExtrasPlugin.class);
-    }
-
     @Override
     protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
         mapperService.merge("_doc", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("_doc",
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
similarity index 100%
rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
rename to core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
index 81056355596..0c20153675a 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
@@ -16,10 +16,10 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.apache.lucene.document.DoubleRange;
 import org.apache.lucene.document.FloatRange;
 import org.apache.lucene.document.InetAddressPoint;
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java
index d91d2b28df8..2b249a5fe6e 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java
@@ -33,9 +33,6 @@ public class MapperExtrasPlugin extends Plugin implements MapperPlugin {
         Map mappers = new LinkedHashMap<>();
         mappers.put(ScaledFloatFieldMapper.CONTENT_TYPE, new ScaledFloatFieldMapper.TypeParser());
         mappers.put(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser());
-        for (RangeFieldMapper.RangeType type : RangeFieldMapper.RangeType.values()) {
-            mappers.put(type.typeName(), new RangeFieldMapper.TypeParser(type));
-        }
         return Collections.unmodifiableMap(mappers);
     }
 
diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle
index 36b93fd4d86..066ae7bb1e0 100644
--- a/modules/percolator/build.gradle
+++ b/modules/percolator/build.gradle
@@ -24,8 +24,6 @@ esplugin {
 }
 
 dependencies {
-    // for testing hasChild and hasParent rejections
-    compile project(path: ':modules:mapper-extras', configuration: 'runtime')
     testCompile project(path: ':modules:parent-join', configuration: 'runtime')
 }
 
diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/range/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/range/10_basic.yml
similarity index 100%
rename from modules/mapper-extras/src/test/resources/rest-api-spec/test/range/10_basic.yml
rename to rest-api-spec/src/main/resources/rest-api-spec/test/range/10_basic.yml

From e6da564eb15945289c1fd0d29d9cd04e3090d51b Mon Sep 17 00:00:00 2001
From: David Turner 
Date: Mon, 18 Dec 2017 08:50:40 +0000
Subject: [PATCH 290/297] Handle case where the hole vertex is south of the
 containing polygon(s) (#27685)

Normally the hole is assigned to the component of the first edge to the south
of one of its vertices, but if the chosen hole vertex is south of everything
then the binary search returns -1 yielding an ArrayIndexOutOfBoundsException.
Instead, assign the vertex to the component of the first edge to its north.
Subsequent validation catches the fact that the hole is outside its component.

Fixes #25933
---
 .../common/geo/builders/PolygonBuilder.java   | 54 +++++++++++++++----
 .../geo/builders/PolygonBuilderTests.java     | 21 +++++++-
 2 files changed, 65 insertions(+), 10 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
index ffcb44c9e46..b0b37dbafa9 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
@@ -469,20 +469,56 @@ public class PolygonBuilder extends ShapeBuilder {
             LOGGER.debug("Holes: {}", Arrays.toString(holes));
         }
         for (int i = 0; i < numHoles; i++) {
+            // To do the assignment we assume (and later, elsewhere, check) that each hole is within
+            // a single component, and the components do not overlap. Based on this assumption, it's
+            // enough to find a component that contains some vertex of the hole, and
+            // holes[i].coordinate is such a vertex, so we use that one.
+
+            // First, we sort all the edges according to their order of intersection with the line
+            // of longitude through holes[i].coordinate, in order from south to north. Edges that do
+            // not intersect this line are sorted to the end of the array and of no further interest
+            // here.
             final Edge current = new Edge(holes[i].coordinate, holes[i].next);
-            // the edge intersects with itself at its own coordinate.  We need intersect to be set this way so the binary search
-            // will get the correct position in the edge list and therefore the correct component to add the hole
             current.intersect = current.coordinate;
             final int intersections = intersections(current.coordinate.x, edges);
-            // if no intersection is found then the hole is not within the polygon, so
-            // don't waste time calling a binary search
-            final int pos;
-            boolean sharedVertex = false;
-            if (intersections == 0 || ((pos = Arrays.binarySearch(edges, 0, intersections, current, INTERSECTION_ORDER)) >= 0)
-                            && !(sharedVertex = (edges[pos].intersect.compareTo(current.coordinate) == 0)) ) {
+
+            if (intersections == 0) {
+                // There were no edges that intersect the line of longitude through
+                // holes[i].coordinate, so there's no way this hole is within the polygon.
                 throw new InvalidShapeException("Invalid shape: Hole is not within polygon");
             }
-            final int index = -((sharedVertex) ? 0 : pos+2);
+
+            // Next we do a binary search to find the position of holes[i].coordinate in the array.
+            // The binary search returns the index of an exact match, or (-insertionPoint - 1) if
+            // the vertex lies between the intersections of edges[insertionPoint] and
+            // edges[insertionPoint+1]. The latter case is vastly more common.
+
+            final int pos;
+            boolean sharedVertex = false;
+            if (((pos = Arrays.binarySearch(edges, 0, intersections, current, INTERSECTION_ORDER)) >= 0)
+                && !(sharedVertex = (edges[pos].intersect.compareTo(current.coordinate) == 0))) {
+                // The binary search returned an exact match, but we checked again using compareTo()
+                // and it didn't match after all.
+
+                // TODO Can this actually happen? Needs a test to exercise it, or else needs to be removed.
+                throw new InvalidShapeException("Invalid shape: Hole is not within polygon");
+            }
+
+            final int index;
+            if (sharedVertex) {
+                // holes[i].coordinate lies exactly on an edge.
+                index = 0; // TODO Should this be pos instead of 0? This assigns exact matches to the southernmost component.
+            } else if (pos == -1) {
+                // holes[i].coordinate is strictly south of all intersections. Assign it to the
+                // southernmost component, and allow later validation to spot that it is not
+                // entirely within the chosen component.
+                index = 0;
+            } else {
+                // holes[i].coordinate is strictly north of at least one intersection. Assign it to
+                // the component immediately to its south.
+                index = -(pos + 2);
+            }
+
             final int component = -edges[index].component - numHoles - 1;
 
             if(debugEnabled()) {
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
index 04122229bc0..8501760d1e7 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java
@@ -20,10 +20,10 @@
 package org.elasticsearch.common.geo.builders;
 
 import com.vividsolutions.jts.geom.Coordinate;
-
 import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType;
+import org.locationtech.spatial4j.exception.InvalidShapeException;
 
 import java.io.IOException;
 
@@ -124,4 +124,23 @@ public class PolygonBuilderTests extends AbstractShapeBuilderTestCase {
+            PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder().coordinate(4, 3).coordinate(3, 2).coordinate(3, 3).close());
+            pb.hole(new LineStringBuilder(new CoordinatesBuilder().coordinate(4, 2).coordinate(3, 1).coordinate(4, 1).close()));
+            pb.build();
+        });
+
+        assertEquals("Hole lies outside shell at or near point (4.0, 1.0, NaN)", e.getMessage());
+    }
+
+    public void testHoleThatIsNorthOfPolygon() {
+        InvalidShapeException e = expectThrows(InvalidShapeException.class, () -> {
+            PolygonBuilder pb = new PolygonBuilder(new CoordinatesBuilder().coordinate(3, 2).coordinate(4, 1).coordinate(3, 1).close());
+            pb.hole(new LineStringBuilder(new CoordinatesBuilder().coordinate(3, 3).coordinate(4, 2).coordinate(4, 3).close()));
+            pb.build();
+        });
+
+        assertEquals("Hole lies outside shell at or near point (4.0, 3.0, NaN)", e.getMessage());
+    }
 }

From 55b71a871b1a4f0f3e6d3c5977c26779819b714a Mon Sep 17 00:00:00 2001
From: Jim Ferenczi 
Date: Mon, 18 Dec 2017 10:36:21 +0100
Subject: [PATCH 291/297] Adapt rest test after backport. Relates #27833

---
 .../rest-api-spec/test/search/110_field_collapsing.yml        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
index 48ca92c1ee9..1ae9c48e59c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
@@ -339,8 +339,8 @@ setup:
 "field collapsing, inner_hits and version":
 
   - skip:
-      version: " - 6.99.99"
-      reason:  "bug fixed in 7.0.0"
+      version: " - 6.1.0"
+      reason:  "bug fixed in 6.1.1"
 
   - do:
       search:

From 7a27a2770bda55ab501768214605b3ceb1f59d2c Mon Sep 17 00:00:00 2001
From: kel 
Date: Mon, 18 Dec 2017 17:38:41 +0800
Subject: [PATCH 292/297] Reject scroll query if size is 0 (#22552) (#27842)

---
 .../action/search/SearchRequest.java          | 12 +++++++----
 .../search/SearchRequestTests.java            | 11 +++++++++-
 .../rest-api-spec/test/scroll/10_basic.yml    | 21 ++++++++++++++++++-
 3 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 87cb645a855..07d2229a536 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -161,17 +161,21 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (source != null && source.trackTotalHits() == false && scroll() != null) {
+        final Scroll scroll = scroll();
+        if (source != null && source.trackTotalHits() == false && scroll != null) {
             validationException =
                 addValidationError("disabling [track_total_hits] is not allowed in a scroll context", validationException);
         }
-        if (source != null && source.from() > 0 &&  scroll() != null) {
+        if (source != null && source.from() > 0 && scroll != null) {
             validationException =
                 addValidationError("using [from] is not allowed in a scroll context", validationException);
         }
-        if (requestCache != null && requestCache && scroll() != null) {
+        if (requestCache != null && requestCache && scroll != null) {
             validationException =
-                addValidationError("[request_cache] cannot be used in a a scroll context", validationException);
+                addValidationError("[request_cache] cannot be used in a scroll context", validationException);
+        }
+        if (source != null && source.size() == 0 && scroll != null) {
+            validationException = addValidationError("[size] cannot be [0] in a scroll context", validationException);
         }
         return validationException;
     }
diff --git a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
index eb643885e83..da8a31d23e1 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
@@ -82,7 +82,6 @@ public class SearchRequestTests extends AbstractSearchTestCase {
     }
 
     public void testValidate() throws IOException {
-
         {
             // if scroll isn't set, validate should never add errors
             SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder());
@@ -114,6 +113,16 @@ public class SearchRequestTests extends AbstractSearchTestCase {
             assertEquals(1, validationErrors.validationErrors().size());
             assertEquals("using [from] is not allowed in a scroll context", validationErrors.validationErrors().get(0));
         }
+        {
+            // scroll and `size` is `0`
+            SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder().size(0));
+            searchRequest.requestCache(false);
+            searchRequest.scroll(new TimeValue(1000));
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNotNull(validationErrors);
+            assertEquals(1, validationErrors.validationErrors().size());
+            assertEquals("[size] cannot be [0] in a scroll context", validationErrors.validationErrors().get(0));
+        }
     }
 
     public void testEqualsAndHashcode() throws IOException {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml
index b7fd64770d3..0ea2779fa92 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml
@@ -206,7 +206,7 @@
       indices.create:
         index: test_scroll
   - do:
-      catch: /\[request_cache\] cannot be used in a a scroll context/
+      catch: /\[request_cache\] cannot be used in a scroll context/
       search:
         index: test_scroll
         scroll: 1m
@@ -214,3 +214,22 @@
         body:
           query:
             match_all: {}
+
+---
+"Scroll with size 0":
+  - skip:
+      version: " - 6.99.99"
+      reason:  the error message has been added in v7.0.0
+  - do:
+      indices.create:
+        index: test_scroll
+  - do:
+      catch: /\[size\] cannot be \[0\] in a scroll context/
+      search:
+        index: test_scroll
+        scroll: 1m
+        request_cache: true
+        body:
+          query:
+            match_all: {}
+          size: 0

From f0b21e3182a56f5424ae2924ea96f49769b0b318 Mon Sep 17 00:00:00 2001
From: David Turner 
Date: Mon, 18 Dec 2017 09:57:40 +0000
Subject: [PATCH 293/297] Make randomNonNegativeLong() draw from a uniform
 distribution (#27856)

Currently randomNonNegativeLong() returns 0 half as often as any positive long,
but random number generators are typically expected to return
uniformly-distributed values unless otherwise specified. This fixes this issue
by mapping Long.MIN_VALUE directly onto 0 rather than resampling.
---
 .../main/java/org/elasticsearch/test/ESTestCase.java   | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 84e750f6e28..d2712e268ad 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -509,12 +509,12 @@ public abstract class ESTestCase extends LuceneTestCase {
         return random().nextInt();
     }
 
+    /**
+     * @return a long between 0 and Long.MAX_VALUE (inclusive) chosen uniformly at random.
+     */
     public static long randomNonNegativeLong() {
-        long randomLong;
-        do {
-            randomLong = randomLong();
-        } while (randomLong == Long.MIN_VALUE);
-        return Math.abs(randomLong);
+        long randomLong = randomLong();
+        return randomLong == Long.MIN_VALUE ? 0 : Math.abs(randomLong);
     }
 
     public static float randomFloat() {

From 26fc717ddd945e1bf9aa34512b75ccb409b38f49 Mon Sep 17 00:00:00 2001
From: kel 
Date: Mon, 18 Dec 2017 18:48:38 +0800
Subject: [PATCH 294/297] Remove unused class PreBuiltTokenFilters (#27839)

---
 .../analysis/PreBuiltTokenFilters.java        | 103 ------------------
 1 file changed, 103 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java

diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java
deleted file mode 100644
index ba66c41e639..00000000000
--- a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.indices.analysis;
-
-import org.apache.lucene.analysis.LowerCaseFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.elasticsearch.Version;
-import org.elasticsearch.index.analysis.MultiTermAwareComponent;
-import org.elasticsearch.index.analysis.TokenFilterFactory;
-import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy;
-
-import java.util.Locale;
-
-public enum PreBuiltTokenFilters {
-    // TODO remove this entire class when PreBuiltTokenizers no longer needs it.....
-    LOWERCASE(CachingStrategy.LUCENE) {
-        @Override
-        public TokenStream create(TokenStream tokenStream, Version version) {
-            return new LowerCaseFilter(tokenStream);
-        }
-        @Override
-        protected boolean isMultiTermAware() {
-            return true;
-        }
-    };
-
-    protected boolean isMultiTermAware() {
-        return false;
-    }
-
-    public abstract TokenStream create(TokenStream tokenStream, Version version);
-
-    protected final PreBuiltCacheFactory.PreBuiltCache cache;
-
-
-    private final CachingStrategy cachingStrategy;
-    PreBuiltTokenFilters(CachingStrategy cachingStrategy) {
-        this.cachingStrategy = cachingStrategy;
-        cache = PreBuiltCacheFactory.getCache(cachingStrategy);
-    }
-
-    public CachingStrategy getCachingStrategy() {
-        return cachingStrategy;
-    }
-
-    private interface MultiTermAwareTokenFilterFactory extends TokenFilterFactory, MultiTermAwareComponent {}
-
-    public synchronized TokenFilterFactory getTokenFilterFactory(final Version version) {
-        TokenFilterFactory factory = cache.get(version);
-        if (factory == null) {
-            final String finalName = name().toLowerCase(Locale.ROOT);
-            if (isMultiTermAware()) {
-                factory = new MultiTermAwareTokenFilterFactory() {
-                    @Override
-                    public String name() {
-                        return finalName;
-                    }
-
-                    @Override
-                    public TokenStream create(TokenStream tokenStream) {
-                        return PreBuiltTokenFilters.this.create(tokenStream, version);
-                    }
-
-                    @Override
-                    public Object getMultiTermComponent() {
-                        return this;
-                    }
-                };
-            } else {
-                factory = new TokenFilterFactory() {
-                    @Override
-                    public String name() {
-                        return finalName;
-                    }
-
-                    @Override
-                    public TokenStream create(TokenStream tokenStream) {
-                        return PreBuiltTokenFilters.this.create(tokenStream, version);
-                    }
-                };
-            }
-            cache.put(version, factory);
-        }
-
-        return factory;
-    }
-}

From 9cd69e7ec196b6fa3ad713414b8fe538c8bdd57c Mon Sep 17 00:00:00 2001
From: Boaz Leskes 
Date: Mon, 18 Dec 2017 13:33:39 +0100
Subject: [PATCH 295/297] recovery from snapshot should fill gaps (#27850)

When snapshotting the primary we capture a lucene commit at an arbitrary moment from a sequence number perspective. This means that it is possible that the commit misses operations and that there is a gap between the local checkpoint in the commit and the maximum sequence number.

When we restore, this will create a primary that "misses" operations and currently will mean that the sequence number system is stuck (i.e., the local checkpoint will be stuck). To fix this we should fill in gaps when we restore, in a similar fashion to normal store recovery.
---
 .../index/shard/StoreRecovery.java            |  2 +
 .../SharedClusterSnapshotRestoreIT.java       | 71 +++++++++++++++++++
 .../elasticsearch/test/ESIntegTestCase.java   |  7 ++
 3 files changed, 80 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index 9216075e822..cfa2ff3dc28 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -436,6 +436,8 @@ final class StoreRecovery {
             final IndexId indexId = repository.getRepositoryData().resolveIndexId(indexName);
             repository.restoreShard(indexShard, restoreSource.snapshot().getSnapshotId(), restoreSource.version(), indexId, snapshotShardId, indexShard.recoveryState());
             indexShard.skipTranslogRecovery();
+            assert indexShard.shardRouting.primary() : "only primary shards can recover from store";
+            indexShard.getEngine().fillSeqNoGaps(indexShard.getPrimaryTerm());
             indexShard.finalizeRecovery();
             indexShard.postRecovery("restore done");
         } catch (Exception e) {
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 45ec0746a9b..7d6bf0ff912 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -36,6 +36,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -71,8 +72,10 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexNameException;
@@ -112,6 +115,7 @@ import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAlloc
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.shard.IndexShardTests.getEngineFromShard;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAliasesExist;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAliasesMissing;
@@ -3072,6 +3076,73 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         }
     }
 
+    public void testSnapshottingWithMissingSequenceNumbers() {
+        final String repositoryName = "test-repo";
+        final String snapshotName = "test-snap";
+        final String indexName = "test-idx";
+        final Client client = client();
+        final Path repo = randomRepoPath();
+
+        logger.info("-->  creating repository at {}", repo.toAbsolutePath());
+        assertAcked(client.admin().cluster().preparePutRepository(repositoryName)
+            .setType("fs").setSettings(Settings.builder()
+                .put("location", repo)
+                .put("compress", false)
+                .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
+        logger.info("--> creating an index and indexing documents");
+        final String dataNode = internalCluster().getDataNodeInstance(ClusterService.class).localNode().getName();
+        final Settings settings =
+            Settings
+                .builder()
+                .put("index.number_of_shards", 1)
+                .put("index.number_of_replicas", 0)
+                .put("index.routing.allocation.include._name", dataNode)
+                .build();
+        createIndex(indexName, settings);
+        ensureGreen();
+        for (int i = 0; i < 5; i++) {
+            index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i);
+        }
+
+        final Index index = resolveIndex(indexName);
+        final IndexShard primary = internalCluster().getInstance(IndicesService.class, dataNode).getShardOrNull(new ShardId(index, 0));
+        // create a gap in the sequence numbers
+        getEngineFromShard(primary).seqNoService().generateSeqNo();
+
+        for (int i = 5; i < 10; i++) {
+            index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i);
+        }
+
+        refresh();
+
+        logger.info("--> snapshot");
+        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshotName)
+            .setWaitForCompletion(true).setIndices(indexName).get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
+            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+
+        logger.info("--> delete indices");
+        assertAcked(client.admin().indices().prepareDelete(indexName));
+
+        logger.info("--> restore all indices from the snapshot");
+        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
+            .setWaitForCompletion(true).execute().actionGet();
+        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
+
+        logger.info("--> indexing some more");
+        for (int i = 10; i < 15; i++) {
+            index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i);
+        }
+
+        IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().get();
+        ShardStats shardStats = stats.getShards()[0];
+        assertTrue(shardStats.getShardRouting().primary());
+        assertThat(shardStats.getSeqNoStats().getLocalCheckpoint(), equalTo(15L)); // 15 indexed docs and one "missing" op.
+        assertThat(shardStats.getSeqNoStats().getGlobalCheckpoint(), equalTo(15L));
+        assertThat(shardStats.getSeqNoStats().getMaxSeqNo(), equalTo(15L));
+    }
+
     private void verifySnapshotInfo(final GetSnapshotsResponse response, final Map> indicesPerSnapshot) {
         for (SnapshotInfo snapshotInfo : response.getSnapshots()) {
             final List expected = snapshotInfo.indices();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index be6a1b29681..e633f5adb70 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -738,6 +738,13 @@ public abstract class ESIntegTestCase extends ESTestCase {
         }
     }
 
+    /**
+     * creates an index with the given setting
+     */
+    public final void createIndex(String name, Settings indexSettings) {
+        assertAcked(prepareCreate(name).setSettings(indexSettings));
+    }
+
     /**
      * Creates a new {@link CreateIndexRequestBuilder} with the settings obtained from {@link #indexSettings()}.
      */

From 76771242e8fdbbc81bced24634e2fad06fa9ff34 Mon Sep 17 00:00:00 2001
From: Jason Tedor 
Date: Mon, 18 Dec 2017 08:19:35 -0500
Subject: [PATCH 296/297] Fix version tests for release tests

This commit fixes the version tests for release tests. The problem here
is that during release tests all version should be treated as released
so the assertions must be modified accordingly.

Relates #27815
---
 .../java/org/elasticsearch/VersionTests.java  |   8 +-
 .../org/elasticsearch/test/VersionUtils.java  |   6 +
 .../elasticsearch/test/VersionUtilsTests.java | 224 +++++++++++++++---
 3 files changed, 200 insertions(+), 38 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/VersionTests.java b/core/src/test/java/org/elasticsearch/VersionTests.java
index 693a2cf9a91..13884014c5e 100644
--- a/core/src/test/java/org/elasticsearch/VersionTests.java
+++ b/core/src/test/java/org/elasticsearch/VersionTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch;
 
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
@@ -339,8 +340,11 @@ public class VersionTests extends ESTestCase {
         assertFalse(isCompatible(Version.fromId(2000099), Version.V_5_0_0));
         assertFalse(isCompatible(Version.fromString("6.0.0"), Version.fromString("7.0.0")));
         assertFalse(isCompatible(Version.fromString("6.0.0-alpha1"), Version.fromString("7.0.0")));
-        assertFalse("only compatible with the latest minor",
-            isCompatible(VersionUtils.getPreviousMinorVersion(), Version.fromString("7.0.0")));
+        final boolean buildSnapshot = Booleans.parseBoolean(System.getProperty("build.snapshot", "true"));
+        assertThat(
+                "[" + VersionUtils.getPreviousMinorVersion() + "] should" + (!buildSnapshot ? " not" : "") + " be compatible with 7.0.0",
+                isCompatible(VersionUtils.getPreviousMinorVersion(), Version.fromString("7.0.0")),
+                equalTo(!buildSnapshot));
         assertFalse(isCompatible(Version.V_5_0_0, Version.fromString("6.0.0")));
         assertFalse(isCompatible(Version.V_5_0_0, Version.fromString("7.0.0")));
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java
index 8fb5f7b81fa..d452b33f577 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.test;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
 
@@ -47,6 +48,11 @@ public class VersionUtils {
      */
     static Tuple, List> resolveReleasedVersions(Version current, Class versionClass) {
         List versions = Version.getDeclaredVersions(versionClass);
+
+        if (!Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) {
+            return Tuple.tuple(versions, Collections.emptyList());
+        }
+
         Version last = versions.remove(versions.size() - 1);
         assert last.equals(current) : "The highest version must be the current one "
             + "but was [" + last + "] and current was [" + current + "]";
diff --git a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
index 7c453c895c6..5312db949b3 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
@@ -19,16 +19,19 @@
 package org.elasticsearch.test;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.collect.Tuple;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
 
 import static java.util.Collections.singletonList;
 import static java.util.stream.Collectors.toCollection;
 import static java.util.stream.Collectors.toList;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
@@ -94,11 +97,13 @@ public class VersionUtilsTests extends ESTestCase {
         got = VersionUtils.randomVersionBetween(random(), Version.CURRENT, null);
         assertEquals(got, Version.CURRENT);
 
-        // max or min can be an unreleased version
-        Version unreleased = randomFrom(VersionUtils.allUnreleasedVersions());
-        assertThat(VersionUtils.randomVersionBetween(random(), null, unreleased), lessThanOrEqualTo(unreleased));
-        assertThat(VersionUtils.randomVersionBetween(random(), unreleased, null), greaterThanOrEqualTo(unreleased));
-        assertEquals(unreleased, VersionUtils.randomVersionBetween(random(), unreleased, unreleased));
+        if (Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) {
+            // max or min can be an unreleased version
+            final Version unreleased = randomFrom(VersionUtils.allUnreleasedVersions());
+            assertThat(VersionUtils.randomVersionBetween(random(), null, unreleased), lessThanOrEqualTo(unreleased));
+            assertThat(VersionUtils.randomVersionBetween(random(), unreleased, null), greaterThanOrEqualTo(unreleased));
+            assertEquals(unreleased, VersionUtils.randomVersionBetween(random(), unreleased, unreleased));
+        }
     }
 
     public static class TestReleaseBranch {
@@ -113,9 +118,28 @@ public class VersionUtilsTests extends ESTestCase {
         Tuple, List> t = VersionUtils.resolveReleasedVersions(TestReleaseBranch.CURRENT, TestReleaseBranch.class);
         List released = t.v1();
         List unreleased = t.v2();
-        assertEquals(Arrays.asList(TestReleaseBranch.V_5_3_0, TestReleaseBranch.V_5_3_1, TestReleaseBranch.V_5_3_2,
-                TestReleaseBranch.V_5_4_0), released);
-        assertEquals(singletonList(TestReleaseBranch.V_5_4_1), unreleased);
+
+        final List expectedReleased;
+        final List expectedUnreleased;
+        if (Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) {
+            expectedReleased = Arrays.asList(
+                    TestReleaseBranch.V_5_3_0,
+                    TestReleaseBranch.V_5_3_1,
+                    TestReleaseBranch.V_5_3_2,
+                    TestReleaseBranch.V_5_4_0);
+            expectedUnreleased = Collections.singletonList(TestReleaseBranch.V_5_4_1);
+        } else {
+            expectedReleased = Arrays.asList(
+                    TestReleaseBranch.V_5_3_0,
+                    TestReleaseBranch.V_5_3_1,
+                    TestReleaseBranch.V_5_3_2,
+                    TestReleaseBranch.V_5_4_0,
+                    TestReleaseBranch.V_5_4_1);
+            expectedUnreleased = Collections.emptyList();
+        }
+
+        assertThat(released, equalTo(expectedReleased));
+        assertThat(unreleased, equalTo(expectedUnreleased));
     }
 
     public static class TestStableBranch {
@@ -130,8 +154,20 @@ public class VersionUtilsTests extends ESTestCase {
                 TestStableBranch.class);
         List released = t.v1();
         List unreleased = t.v2();
-        assertEquals(Arrays.asList(TestStableBranch.V_5_3_0, TestStableBranch.V_5_3_1), released);
-        assertEquals(Arrays.asList(TestStableBranch.V_5_3_2, TestStableBranch.V_5_4_0), unreleased);
+
+        final List expectedReleased;
+        final List expectedUnreleased;
+        if (Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) {
+            expectedReleased = Arrays.asList(TestStableBranch.V_5_3_0, TestStableBranch.V_5_3_1);
+            expectedUnreleased = Arrays.asList(TestStableBranch.V_5_3_2, TestStableBranch.V_5_4_0);
+        } else {
+            expectedReleased =
+                    Arrays.asList(TestStableBranch.V_5_3_0, TestStableBranch.V_5_3_1, TestStableBranch.V_5_3_2, TestStableBranch.V_5_4_0);
+            expectedUnreleased = Collections.emptyList();
+        }
+
+        assertThat(released, equalTo(expectedReleased));
+        assertThat(unreleased, equalTo(expectedUnreleased));
     }
 
     public static class TestStableBranchBehindStableBranch {
@@ -147,9 +183,26 @@ public class VersionUtilsTests extends ESTestCase {
                 TestStableBranchBehindStableBranch.class);
         List released = t.v1();
         List unreleased = t.v2();
-        assertEquals(Arrays.asList(TestStableBranchBehindStableBranch.V_5_3_0, TestStableBranchBehindStableBranch.V_5_3_1), released);
-        assertEquals(Arrays.asList(TestStableBranchBehindStableBranch.V_5_3_2, TestStableBranchBehindStableBranch.V_5_4_0,
-                TestStableBranchBehindStableBranch.V_5_5_0), unreleased);
+
+        final List expectedReleased;
+        final List expectedUnreleased;
+        if (Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) {
+            expectedReleased = Arrays.asList(TestStableBranchBehindStableBranch.V_5_3_0, TestStableBranchBehindStableBranch.V_5_3_1);
+            expectedUnreleased = Arrays.asList(
+                    TestStableBranchBehindStableBranch.V_5_3_2,
+                    TestStableBranchBehindStableBranch.V_5_4_0,
+                    TestStableBranchBehindStableBranch.V_5_5_0);
+        } else {
+            expectedReleased = Arrays.asList(
+                    TestStableBranchBehindStableBranch.V_5_3_0,
+                    TestStableBranchBehindStableBranch.V_5_3_1,
+                    TestStableBranchBehindStableBranch.V_5_3_2,
+                    TestStableBranchBehindStableBranch.V_5_4_0,
+                    TestStableBranchBehindStableBranch.V_5_5_0);
+            expectedUnreleased = Collections.emptyList();
+        }
+        assertThat(released, equalTo(expectedReleased));
+        assertThat(unreleased, equalTo(expectedUnreleased));
     }
 
     public static class TestUnstableBranch {
@@ -168,9 +221,29 @@ public class VersionUtilsTests extends ESTestCase {
                 TestUnstableBranch.class);
         List released = t.v1();
         List unreleased = t.v2();
-        assertEquals(Arrays.asList(TestUnstableBranch.V_5_3_0, TestUnstableBranch.V_5_3_1,
-                TestUnstableBranch.V_6_0_0_alpha1, TestUnstableBranch.V_6_0_0_alpha2), released);
-        assertEquals(Arrays.asList(TestUnstableBranch.V_5_3_2, TestUnstableBranch.V_5_4_0, TestUnstableBranch.V_6_0_0_beta1), unreleased);
+
+        final List expectedReleased;
+        final List expectedUnreleased;
+        if (Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) {
+            expectedReleased = Arrays.asList(
+                    TestUnstableBranch.V_5_3_0,
+                    TestUnstableBranch.V_5_3_1,
+                    TestUnstableBranch.V_6_0_0_alpha1,
+                    TestUnstableBranch.V_6_0_0_alpha2);
+            expectedUnreleased = Arrays.asList(TestUnstableBranch.V_5_3_2, TestUnstableBranch.V_5_4_0, TestUnstableBranch.V_6_0_0_beta1);
+        } else {
+            expectedReleased = Arrays.asList(
+                    TestUnstableBranch.V_5_3_0,
+                    TestUnstableBranch.V_5_3_1,
+                    TestUnstableBranch.V_5_3_2,
+                    TestUnstableBranch.V_5_4_0,
+                    TestUnstableBranch.V_6_0_0_alpha1,
+                    TestUnstableBranch.V_6_0_0_alpha2,
+                    TestUnstableBranch.V_6_0_0_beta1);
+            expectedUnreleased = Collections.emptyList();
+        }
+        assertThat(released, equalTo(expectedReleased));
+        assertThat(unreleased, equalTo(expectedUnreleased));
     }
 
     public static class TestNewMajorRelease {
@@ -191,11 +264,35 @@ public class VersionUtilsTests extends ESTestCase {
             TestNewMajorRelease.class);
         List released = t.v1();
         List unreleased = t.v2();
-        assertEquals(Arrays.asList(TestNewMajorRelease.V_5_6_0, TestNewMajorRelease.V_5_6_1,
-            TestNewMajorRelease.V_6_0_0_alpha1, TestNewMajorRelease.V_6_0_0_alpha2,
-            TestNewMajorRelease.V_6_0_0_beta1, TestNewMajorRelease.V_6_0_0_beta2,
-            TestNewMajorRelease.V_6_0_0), released);
-        assertEquals(Arrays.asList(TestNewMajorRelease.V_5_6_2, TestNewMajorRelease.V_6_0_1), unreleased);
+
+        final List expectedReleased;
+        final List expectedUnreleased;
+        if (Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) {
+            expectedReleased = Arrays.asList(
+                    TestNewMajorRelease.V_5_6_0,
+                    TestNewMajorRelease.V_5_6_1,
+                    TestNewMajorRelease.V_6_0_0_alpha1,
+                    TestNewMajorRelease.V_6_0_0_alpha2,
+                    TestNewMajorRelease.V_6_0_0_beta1,
+                    TestNewMajorRelease.V_6_0_0_beta2,
+                    TestNewMajorRelease.V_6_0_0);
+            expectedUnreleased = Arrays.asList(TestNewMajorRelease.V_5_6_2, TestNewMajorRelease.V_6_0_1);
+        } else {
+            expectedReleased = Arrays.asList(
+                    TestNewMajorRelease.V_5_6_0,
+                    TestNewMajorRelease.V_5_6_1,
+                    TestNewMajorRelease.V_5_6_2,
+                    TestNewMajorRelease.V_6_0_0_alpha1,
+                    TestNewMajorRelease.V_6_0_0_alpha2,
+                    TestNewMajorRelease.V_6_0_0_beta1,
+                    TestNewMajorRelease.V_6_0_0_beta2,
+                    TestNewMajorRelease.V_6_0_0,
+                    TestNewMajorRelease.V_6_0_1);
+            expectedUnreleased = Collections.emptyList();
+        }
+
+        assertThat(released, equalTo(expectedReleased));
+        assertThat(unreleased, equalTo(expectedUnreleased));
     }
 
     public static class TestVersionBumpIn6x {
@@ -217,11 +314,37 @@ public class VersionUtilsTests extends ESTestCase {
             TestVersionBumpIn6x.class);
         List released = t.v1();
         List unreleased = t.v2();
-        assertEquals(Arrays.asList(TestVersionBumpIn6x.V_5_6_0, TestVersionBumpIn6x.V_5_6_1,
-            TestVersionBumpIn6x.V_6_0_0_alpha1, TestVersionBumpIn6x.V_6_0_0_alpha2,
-            TestVersionBumpIn6x.V_6_0_0_beta1, TestVersionBumpIn6x.V_6_0_0_beta2,
-            TestVersionBumpIn6x.V_6_0_0), released);
-        assertEquals(Arrays.asList(TestVersionBumpIn6x.V_5_6_2, TestVersionBumpIn6x.V_6_0_1, TestVersionBumpIn6x.V_6_1_0), unreleased);
+
+        final List expectedReleased;
+        final List expectedUnreleased;
+
+        if (Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) {
+            expectedReleased = Arrays.asList(
+                    TestVersionBumpIn6x.V_5_6_0,
+                    TestVersionBumpIn6x.V_5_6_1,
+                    TestVersionBumpIn6x.V_6_0_0_alpha1,
+                    TestVersionBumpIn6x.V_6_0_0_alpha2,
+                    TestVersionBumpIn6x.V_6_0_0_beta1,
+                    TestVersionBumpIn6x.V_6_0_0_beta2,
+                    TestVersionBumpIn6x.V_6_0_0);
+            expectedUnreleased = Arrays.asList(TestVersionBumpIn6x.V_5_6_2, TestVersionBumpIn6x.V_6_0_1, TestVersionBumpIn6x.V_6_1_0);
+        } else {
+            expectedReleased = Arrays.asList(
+                    TestVersionBumpIn6x.V_5_6_0,
+                    TestVersionBumpIn6x.V_5_6_1,
+                    TestVersionBumpIn6x.V_5_6_2,
+                    TestVersionBumpIn6x.V_6_0_0_alpha1,
+                    TestVersionBumpIn6x.V_6_0_0_alpha2,
+                    TestVersionBumpIn6x.V_6_0_0_beta1,
+                    TestVersionBumpIn6x.V_6_0_0_beta2,
+                    TestVersionBumpIn6x.V_6_0_0,
+                    TestVersionBumpIn6x.V_6_0_1,
+                    TestVersionBumpIn6x.V_6_1_0);
+            expectedUnreleased = Collections.emptyList();
+        }
+
+        assertThat(released, equalTo(expectedReleased));
+        assertThat(unreleased, equalTo(expectedUnreleased));
     }
 
     public static class TestNewMinorBranchIn6x {
@@ -246,12 +369,45 @@ public class VersionUtilsTests extends ESTestCase {
             TestNewMinorBranchIn6x.class);
         List released = t.v1();
         List unreleased = t.v2();
-        assertEquals(Arrays.asList(TestNewMinorBranchIn6x.V_5_6_0, TestNewMinorBranchIn6x.V_5_6_1,
-            TestNewMinorBranchIn6x.V_6_0_0_alpha1, TestNewMinorBranchIn6x.V_6_0_0_alpha2,
-            TestNewMinorBranchIn6x.V_6_0_0_beta1, TestNewMinorBranchIn6x.V_6_0_0_beta2,
-            TestNewMinorBranchIn6x.V_6_0_0, TestNewMinorBranchIn6x.V_6_1_0, TestNewMinorBranchIn6x.V_6_1_1), released);
-        assertEquals(Arrays.asList(TestNewMinorBranchIn6x.V_5_6_2, TestNewMinorBranchIn6x.V_6_0_1,
-            TestNewMinorBranchIn6x.V_6_1_2, TestNewMinorBranchIn6x.V_6_2_0), unreleased);
+
+        final List expectedReleased;
+        final List expectedUnreleased;
+        if (Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) {
+            expectedReleased = Arrays.asList(
+                    TestNewMinorBranchIn6x.V_5_6_0,
+                    TestNewMinorBranchIn6x.V_5_6_1,
+                    TestNewMinorBranchIn6x.V_6_0_0_alpha1,
+                    TestNewMinorBranchIn6x.V_6_0_0_alpha2,
+                    TestNewMinorBranchIn6x.V_6_0_0_beta1,
+                    TestNewMinorBranchIn6x.V_6_0_0_beta2,
+                    TestNewMinorBranchIn6x.V_6_0_0,
+                    TestNewMinorBranchIn6x.V_6_1_0,
+                    TestNewMinorBranchIn6x.V_6_1_1);
+            expectedUnreleased = Arrays.asList(
+                    TestNewMinorBranchIn6x.V_5_6_2,
+                    TestNewMinorBranchIn6x.V_6_0_1,
+                    TestNewMinorBranchIn6x.V_6_1_2,
+                    TestNewMinorBranchIn6x.V_6_2_0);
+        } else {
+            expectedReleased = Arrays.asList(
+                    TestNewMinorBranchIn6x.V_5_6_0,
+                    TestNewMinorBranchIn6x.V_5_6_1,
+                    TestNewMinorBranchIn6x.V_5_6_2,
+                    TestNewMinorBranchIn6x.V_6_0_0_alpha1,
+                    TestNewMinorBranchIn6x.V_6_0_0_alpha2,
+                    TestNewMinorBranchIn6x.V_6_0_0_beta1,
+                    TestNewMinorBranchIn6x.V_6_0_0_beta2,
+                    TestNewMinorBranchIn6x.V_6_0_0,
+                    TestNewMinorBranchIn6x.V_6_0_1,
+                    TestNewMinorBranchIn6x.V_6_1_0,
+                    TestNewMinorBranchIn6x.V_6_1_1,
+                    TestNewMinorBranchIn6x.V_6_1_2,
+                    TestNewMinorBranchIn6x.V_6_2_0);
+            expectedUnreleased = Collections.emptyList();
+        }
+
+        assertThat(released, equalTo(expectedReleased));
+        assertThat(unreleased, equalTo(expectedUnreleased));
     }
 
     /**
@@ -259,10 +415,6 @@ public class VersionUtilsTests extends ESTestCase {
      * agree with the list of wire and index compatible versions we build in gradle.
      */
     public void testGradleVersionsMatchVersionUtils() {
-        if (System.getProperty("build.snapshot", "true").equals("false")) {
-            logger.warn("Skipping testGradleVersionsMatchVersionUtils(): See #27815 for details");
-            return;
-        }
         // First check the index compatible versions
         VersionsFromProperty indexCompatible = new VersionsFromProperty("tests.gradle_index_compat_versions");
         List released = VersionUtils.allReleasedVersions().stream()

From af3f63616b3822039ecae1cc9142501111904e50 Mon Sep 17 00:00:00 2001
From: Alan Woodward 
Date: Mon, 18 Dec 2017 14:27:03 +0000
Subject: [PATCH 297/297] Allow TrimFilter to be used in custom normalizers
 (#27758)

AnalysisFactoryTestCase checks that the ES custom token filter multi-term
awareness matches the underlying lucene factory.  For the trim filter this
won't be the case until LUCENE-8093 is released in 7.3, so we add a
temporary exclusion

Closes #27310
---
 .../analysis/common/CommonAnalysisPlugin.java |  2 +-
 .../analysis/common/TrimTokenFilterTests.java | 48 +++++++++++++++++++
 .../analysis/AnalysisFactoryTestCase.java     |  6 +++
 3 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TrimTokenFilterTests.java

diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
index 6f47ad5f770..e0193e50313 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
@@ -246,7 +246,7 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin {
         filters.add(PreConfiguredTokenFilter.singleton("stemmer", false, PorterStemFilter::new));
         // The stop filter is in lucene-core but the English stop words set is in lucene-analyzers-common
         filters.add(PreConfiguredTokenFilter.singleton("stop", false, input -> new StopFilter(input, StopAnalyzer.ENGLISH_STOP_WORDS_SET)));
-        filters.add(PreConfiguredTokenFilter.singleton("trim", false, TrimFilter::new));
+        filters.add(PreConfiguredTokenFilter.singleton("trim", true, TrimFilter::new));
         filters.add(PreConfiguredTokenFilter.singleton("truncate", false, input -> new TruncateTokenFilter(input, 10)));
         filters.add(PreConfiguredTokenFilter.singleton("type_as_payload", false, TypeAsPayloadTokenFilter::new));
         filters.add(PreConfiguredTokenFilter.singleton("unique", false, UniqueTokenFilter::new));
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TrimTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TrimTokenFilterTests.java
new file mode 100644
index 00000000000..2ce9af2b391
--- /dev/null
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TrimTokenFilterTests.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.analysis.AnalysisTestsHelper;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTokenStreamTestCase;
+
+import java.io.IOException;
+
+public class TrimTokenFilterTests extends ESTokenStreamTestCase {
+
+    public void testNormalizer() throws IOException {
+        Settings settings = Settings.builder()
+            .putList("index.analysis.normalizer.my_normalizer.filter", "trim")
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin());
+        assertNull(analysis.indexAnalyzers.get("my_normalizer"));
+        NamedAnalyzer normalizer = analysis.indexAnalyzers.getNormalizer("my_normalizer");
+        assertNotNull(normalizer);
+        assertEquals("my_normalizer", normalizer.name());
+        assertTokenStreamContents(normalizer.tokenStream("foo", "  bar  "), new String[] {"bar"});
+        assertEquals(new BytesRef("bar"), normalizer.normalize("foo", "  bar  "));
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java
index ff67f874fda..29d58ae2577 100644
--- a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java
@@ -22,6 +22,7 @@ package org.elasticsearch.indices.analysis;
 import org.apache.lucene.analysis.util.CharFilterFactory;
 import org.apache.lucene.analysis.util.TokenFilterFactory;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.index.analysis.ClassicTokenizerFactory;
 import org.elasticsearch.index.analysis.EdgeNGramTokenizerFactory;
@@ -462,6 +463,11 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
 
         Set classesThatShouldNotHaveMultiTermSupport = new HashSet<>(actual);
         classesThatShouldNotHaveMultiTermSupport.removeAll(expected);
+        classesThatShouldNotHaveMultiTermSupport.remove("token filter [trim]");
+        if (Version.CURRENT.luceneVersion.onOrAfter(org.apache.lucene.util.Version.fromBits(7, 3, 0))) {
+            // TODO: remove the above exclusion when we move to lucene 7.3
+            assert false;
+        }
         assertTrue("Pre-built components should not have multi-term support: " + classesThatShouldNotHaveMultiTermSupport,
                 classesThatShouldNotHaveMultiTermSupport.isEmpty());
     }