diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index 9e2c0536b6f..0e8bd709ec9 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -400,27 +400,27 @@ public class SearchRequestBuilder extends ActionRequestBuilder rescorer) { sourceBuilder().clearRescorers(); return addRescorer(rescorer); } /** * Clears all rescorers on the builder and sets the first one. To use multiple rescore windows use - * {@link #addRescorer(org.elasticsearch.search.rescore.RescoreBuilder.Rescorer, int)}. + * {@link #addRescorer(org.elasticsearch.search.rescore.RescoreBuilder, int)}. * * @param rescorer rescorer configuration * @param window rescore window * @return this for chaining */ - public SearchRequestBuilder setRescorer(RescoreBuilder.Rescorer rescorer, int window) { + public SearchRequestBuilder setRescorer(RescoreBuilder rescorer, int window) { sourceBuilder().clearRescorers(); - return addRescorer(rescorer, window); + return addRescorer(rescorer.windowSize(window)); } /** @@ -429,8 +429,8 @@ public class SearchRequestBuilder extends ActionRequestBuilder rescorer) { + sourceBuilder().addRescorer(rescorer); return this; } @@ -441,8 +441,8 @@ public class SearchRequestBuilder extends ActionRequestBuilder rescorer, int window) { + sourceBuilder().addRescorer(rescorer.windowSize(window)); return this; } diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 263b8c4a05b..95c6b7c8f16 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -37,7 +37,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; -import org.elasticsearch.search.rescore.RescoreBuilder.Rescorer; +import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorFactory; import org.joda.time.DateTime; @@ -63,7 +63,6 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.ElasticsearchException.readException; @@ -694,10 +693,10 @@ public abstract class StreamInput extends InputStream { } /** - * Reads a {@link QueryBuilder} from the current stream + * Reads a {@link RescoreBuilder} from the current stream */ - public Rescorer readRescorer() throws IOException { - return readNamedWriteable(Rescorer.class); + public RescoreBuilder readRescorer() throws IOException { + return readNamedWriteable(RescoreBuilder.class); } /** diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 591081f3d11..14b02e68803 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -36,7 +36,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; -import org.elasticsearch.search.rescore.RescoreBuilder.Rescorer; +import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorFactory; import org.joda.time.ReadableInstant; @@ -692,12 +692,12 @@ public abstract class StreamOutput extends OutputStream { for (T obj: list) { obj.writeTo(this); } - } + } /** - * Writes a {@link Rescorer} to the current stream + * Writes a {@link RescoreBuilder} to the current stream */ - public void writeRescorer(Rescorer rescorer) throws IOException { + public void writeRescorer(RescoreBuilder rescorer) throws IOException { writeNamedWriteable(rescorer); } } diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java b/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java index 0e507513102..32fade45e9d 100644 --- a/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java +++ b/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java @@ -22,8 +22,6 @@ package org.elasticsearch.ingest; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.collect.HppcMaps; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestService.java b/core/src/main/java/org/elasticsearch/ingest/IngestService.java index bc7cd75070c..8af82b28a38 100644 --- a/core/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/core/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -19,7 +19,6 @@ package org.elasticsearch.ingest; -import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ThreadPool; @@ -28,8 +27,7 @@ import java.io.Closeable; import java.io.IOException; /** - * Instantiates and wires all the services that the ingest plugin will be needing. - * Also the bootstrapper is in charge of starting and stopping the ingest plugin based on the cluster state. + * Holder class for several ingest related services. */ public class IngestService implements Closeable { diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index 90ab2a76c2e..3bd80edc306 100644 --- a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -19,8 +19,6 @@ package org.elasticsearch.ingest; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Build; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -34,7 +32,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.Map; -import java.util.Objects; import java.util.function.BiFunction; /** diff --git a/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java b/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java index bc5fd19aac7..699720e18ca 100644 --- a/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java +++ b/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java @@ -25,7 +25,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; /** * A Processor that executes a list of other "processors". It executes a separate list of diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/DeDotProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/DeDotProcessor.java index 295a9884997..b8f86616ffc 100644 --- a/core/src/main/java/org/elasticsearch/ingest/processor/DeDotProcessor.java +++ b/core/src/main/java/org/elasticsearch/ingest/processor/DeDotProcessor.java @@ -23,7 +23,6 @@ import org.elasticsearch.ingest.core.AbstractProcessor; import org.elasticsearch.ingest.core.AbstractProcessorFactory; import org.elasticsearch.ingest.core.ConfigurationUtils; import org.elasticsearch.ingest.core.IngestDocument; -import org.elasticsearch.ingest.core.Processor; import java.util.HashMap; import java.util.Iterator; diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/FailProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/FailProcessor.java index 65b4b602bd0..76c7b3c40ea 100644 --- a/core/src/main/java/org/elasticsearch/ingest/processor/FailProcessor.java +++ b/core/src/main/java/org/elasticsearch/ingest/processor/FailProcessor.java @@ -23,7 +23,6 @@ import org.elasticsearch.ingest.core.AbstractProcessor; import org.elasticsearch.ingest.core.AbstractProcessorFactory; import org.elasticsearch.ingest.core.ConfigurationUtils; import org.elasticsearch.ingest.core.IngestDocument; -import org.elasticsearch.ingest.core.Processor; import org.elasticsearch.ingest.core.TemplateService; import java.util.Map; diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/GsubProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/GsubProcessor.java index 3dc4b3f0cad..0ec7fba84f2 100644 --- a/core/src/main/java/org/elasticsearch/ingest/processor/GsubProcessor.java +++ b/core/src/main/java/org/elasticsearch/ingest/processor/GsubProcessor.java @@ -23,7 +23,6 @@ import org.elasticsearch.ingest.core.AbstractProcessor; import org.elasticsearch.ingest.core.AbstractProcessorFactory; import org.elasticsearch.ingest.core.IngestDocument; import org.elasticsearch.ingest.core.ConfigurationUtils; -import org.elasticsearch.ingest.core.Processor; import java.util.Map; import java.util.regex.Matcher; diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/JoinProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/JoinProcessor.java index 3516929e26f..dd729dd0afd 100644 --- a/core/src/main/java/org/elasticsearch/ingest/processor/JoinProcessor.java +++ b/core/src/main/java/org/elasticsearch/ingest/processor/JoinProcessor.java @@ -23,7 +23,6 @@ import org.elasticsearch.ingest.core.AbstractProcessor; import org.elasticsearch.ingest.core.AbstractProcessorFactory; import org.elasticsearch.ingest.core.IngestDocument; import org.elasticsearch.ingest.core.ConfigurationUtils; -import org.elasticsearch.ingest.core.Processor; import java.util.List; import java.util.Map; diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/RemoveProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/RemoveProcessor.java index e994954a034..a39ac8f5cf4 100644 --- a/core/src/main/java/org/elasticsearch/ingest/processor/RemoveProcessor.java +++ b/core/src/main/java/org/elasticsearch/ingest/processor/RemoveProcessor.java @@ -24,7 +24,6 @@ import org.elasticsearch.ingest.core.AbstractProcessorFactory; import org.elasticsearch.ingest.core.IngestDocument; import org.elasticsearch.ingest.core.TemplateService; import org.elasticsearch.ingest.core.ConfigurationUtils; -import org.elasticsearch.ingest.core.Processor; import java.util.Map; diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/RenameProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/RenameProcessor.java index 7726a720b49..6088315884e 100644 --- a/core/src/main/java/org/elasticsearch/ingest/processor/RenameProcessor.java +++ b/core/src/main/java/org/elasticsearch/ingest/processor/RenameProcessor.java @@ -23,7 +23,6 @@ import org.elasticsearch.ingest.core.AbstractProcessor; import org.elasticsearch.ingest.core.AbstractProcessorFactory; import org.elasticsearch.ingest.core.IngestDocument; import org.elasticsearch.ingest.core.ConfigurationUtils; -import org.elasticsearch.ingest.core.Processor; import java.util.Map; diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java index 5442fb325d5..6d663a36d73 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java @@ -19,14 +19,6 @@ package org.elasticsearch.search; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; - import org.apache.lucene.search.BooleanQuery; import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.geo.builders.CircleBuilder; @@ -234,10 +226,20 @@ import org.elasticsearch.search.highlight.Highlighter; import org.elasticsearch.search.highlight.HighlighterParseElement; import org.elasticsearch.search.highlight.Highlighters; import org.elasticsearch.search.query.QueryPhase; +import org.elasticsearch.search.rescore.QueryRescorerBuilder; +import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.sort.SortParseElement; import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.Suggesters; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + /** * */ @@ -336,6 +338,7 @@ public class SearchModule extends AbstractModule { configureSuggesters(); configureFetchSubPhase(); configureShapes(); + configureRescorers(); } protected void configureFetchSubPhase() { @@ -468,6 +471,10 @@ public class SearchModule extends AbstractModule { } } + private void configureRescorers() { + namedWriteableRegistry.registerPrototype(RescoreBuilder.class, QueryRescorerBuilder.PROTOTYPE); + } + private void registerBuiltinFunctionScoreParsers() { registerFunctionScoreParser(new ScriptScoreFunctionParser()); registerFunctionScoreParser(new GaussDecayFunctionParser()); diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index a91014dea34..78fccefb5ef 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -61,7 +61,6 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Loading; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.internal.ParentFieldMapper; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.search.stats.ShardSearchStats; @@ -101,6 +100,7 @@ import org.elasticsearch.search.query.QuerySearchRequest; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.query.QuerySearchResultProvider; import org.elasticsearch.search.query.ScrollQuerySearchResult; +import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -776,33 +776,12 @@ public class SearchService extends AbstractLifecycleComponent imp } } if (source.rescores() != null) { - XContentParser completeRescoreParser = null; try { - XContentBuilder completeRescoreBuilder = XContentFactory.jsonBuilder(); - completeRescoreBuilder.startObject(); - completeRescoreBuilder.startArray("rescore"); - for (BytesReference rescore : source.rescores()) { - XContentParser parser = XContentFactory.xContent(rescore).createParser(rescore); - parser.nextToken(); - completeRescoreBuilder.copyCurrentStructure(parser); + for (RescoreBuilder rescore : source.rescores()) { + context.addRescore(rescore.build(context.indexShard().getQueryShardContext())); } - completeRescoreBuilder.endArray(); - completeRescoreBuilder.endObject(); - BytesReference completeRescoreBytes = completeRescoreBuilder.bytes(); - completeRescoreParser = XContentFactory.xContent(completeRescoreBytes).createParser(completeRescoreBytes); - completeRescoreParser.nextToken(); - completeRescoreParser.nextToken(); - completeRescoreParser.nextToken(); - this.elementParsers.get("rescore").parse(completeRescoreParser, context); - } catch (Exception e) { - String sSource = "_na_"; - try { - sSource = source.toString(); - } catch (Throwable e1) { - // ignore - } - XContentLocation location = completeRescoreParser != null ? completeRescoreParser.getTokenLocation() : null; - throw new SearchParseException(context, "failed to parse rescore source [" + sSource + "]", location, e); + } catch (IOException e) { + throw new SearchContextException(context, "failed to create RescoreSearchContext", e); } } if (source.fields() != null) { diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 59ffb0ad6fa..e003b445be3 100644 --- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -48,6 +48,7 @@ import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.rescore.RescoreBuilder; +import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -153,7 +154,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ private BytesReference innerHitsBuilder; - private List rescoreBuilders; + private List> rescoreBuilders; private ObjectFloatHashMap indexBoost = null; @@ -479,20 +480,12 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ return suggestBuilder; } - public SearchSourceBuilder addRescorer(RescoreBuilder rescoreBuilder) { - try { + public SearchSourceBuilder addRescorer(RescoreBuilder rescoreBuilder) { if (rescoreBuilders == null) { rescoreBuilders = new ArrayList<>(); } - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - rescoreBuilder.toXContent(builder, EMPTY_PARAMS); - builder.endObject(); - rescoreBuilders.add(builder.bytes()); + rescoreBuilders.add(rescoreBuilder); return this; - } catch (IOException e) { - throw new RuntimeException(e); - } } public SearchSourceBuilder clearRescorers() { @@ -519,7 +512,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ /** * Gets the bytes representing the rescore builders for this request. */ - public List rescores() { + public List> rescores() { return rescoreBuilders; } @@ -899,10 +892,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } builder.sorts = sorts; } else if (context.parseFieldMatcher().match(currentFieldName, RESCORE_FIELD)) { - List rescoreBuilders = new ArrayList<>(); + List> rescoreBuilders = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); - rescoreBuilders.add(xContentBuilder.bytes()); + rescoreBuilders.add(RescoreBuilder.parseFromXContent(context)); } builder.rescoreBuilders = rescoreBuilders; } else if (context.parseFieldMatcher().match(currentFieldName, STATS_FIELD)) { @@ -1069,10 +1061,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ if (rescoreBuilders != null) { builder.startArray(RESCORE_FIELD.getPreferredName()); - for (BytesReference rescoreBuilder : rescoreBuilders) { - XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(rescoreBuilder); - parser.nextToken(); - builder.copyCurrentStructure(parser); + for (RescoreBuilder rescoreBuilder : rescoreBuilders) { + rescoreBuilder.toXContent(builder, params); } builder.endArray(); } @@ -1218,9 +1208,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } if (in.readBoolean()) { int size = in.readVInt(); - List rescoreBuilders = new ArrayList<>(); + List> rescoreBuilders = new ArrayList<>(); for (int i = 0; i < size; i++) { - rescoreBuilders.add(in.readBytesReference()); + rescoreBuilders.add(in.readRescorer()); } builder.rescoreBuilders = rescoreBuilders; } @@ -1334,8 +1324,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ out.writeBoolean(hasRescoreBuilders); if (hasRescoreBuilders) { out.writeVInt(rescoreBuilders.size()); - for (BytesReference rescoreBuilder : rescoreBuilders) { - out.writeBytesReference(rescoreBuilder); + for (RescoreBuilder rescoreBuilder : rescoreBuilders) { + out.writeRescorer(rescoreBuilder); } } boolean hasScriptFields = scriptFields != null; diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java index 7f95ff10824..319055639ac 100644 --- a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java +++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java @@ -27,7 +27,7 @@ import org.apache.lucene.search.TopDocs; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.SearchContext; @@ -120,17 +120,17 @@ public final class QueryRescorer implements Rescorer { } } - private static final ObjectParser RESCORE_PARSER = new ObjectParser<>("query", null); + private static final ObjectParser RESCORE_PARSER = new ObjectParser<>("query", null); static { - RESCORE_PARSER.declareObject(QueryRescoreContext::setParsedQuery, (p, c) -> c.indexShard().getQueryShardContext().parse(p), new ParseField("rescore_query")); + RESCORE_PARSER.declareObject(QueryRescoreContext::setQuery, (p, c) -> c.parse(p).query(), new ParseField("rescore_query")); RESCORE_PARSER.declareFloat(QueryRescoreContext::setQueryWeight, new ParseField("query_weight")); RESCORE_PARSER.declareFloat(QueryRescoreContext::setRescoreQueryWeight, new ParseField("rescore_query_weight")); RESCORE_PARSER.declareString(QueryRescoreContext::setScoreMode, new ParseField("score_mode")); } @Override - public RescoreSearchContext parse(XContentParser parser, SearchContext context) throws IOException { + public RescoreSearchContext parse(XContentParser parser, QueryShardContext context) throws IOException { return RESCORE_PARSER.parse(parser, new QueryRescoreContext(this), context); } @@ -178,22 +178,24 @@ public final class QueryRescorer implements Rescorer { public static class QueryRescoreContext extends RescoreSearchContext { + static final int DEFAULT_WINDOW_SIZE = 10; + public QueryRescoreContext(QueryRescorer rescorer) { - super(NAME, 10, rescorer); + super(NAME, DEFAULT_WINDOW_SIZE, rescorer); this.scoreMode = QueryRescoreMode.Total; } - private ParsedQuery parsedQuery; + private Query query; private float queryWeight = 1.0f; private float rescoreQueryWeight = 1.0f; private QueryRescoreMode scoreMode; - public void setParsedQuery(ParsedQuery parsedQuery) { - this.parsedQuery = parsedQuery; + public void setQuery(Query query) { + this.query = query; } public Query query() { - return parsedQuery.query(); + return query; } public float queryWeight() { diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java new file mode 100644 index 00000000000..10c727a9029 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java @@ -0,0 +1,242 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.rescore; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.rescore.QueryRescorer.QueryRescoreContext; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +public class QueryRescorerBuilder extends RescoreBuilder { + + public static final String NAME = "query"; + + public static final QueryRescorerBuilder PROTOTYPE = new QueryRescorerBuilder(new MatchAllQueryBuilder()); + + public static final float DEFAULT_RESCORE_QUERYWEIGHT = 1.0f; + public static final float DEFAULT_QUERYWEIGHT = 1.0f; + public static final QueryRescoreMode DEFAULT_SCORE_MODE = QueryRescoreMode.Total; + private final QueryBuilder queryBuilder; + private float rescoreQueryWeight = DEFAULT_RESCORE_QUERYWEIGHT; + private float queryWeight = DEFAULT_QUERYWEIGHT; + private QueryRescoreMode scoreMode = DEFAULT_SCORE_MODE; + + private static ParseField RESCORE_QUERY_FIELD = new ParseField("rescore_query"); + private static ParseField QUERY_WEIGHT_FIELD = new ParseField("query_weight"); + private static ParseField RESCORE_QUERY_WEIGHT_FIELD = new ParseField("rescore_query_weight"); + private static ParseField SCORE_MODE_FIELD = new ParseField("score_mode"); + + private static final ObjectParser QUERY_RESCORE_PARSER = new ObjectParser<>(NAME, null); + + static { + QUERY_RESCORE_PARSER.declareObject(InnerBuilder::setQueryBuilder, (p, c) -> { + try { + return c.parseInnerQueryBuilder(); + } catch (IOException e) { + throw new ParsingException(p.getTokenLocation(), "Could not parse inner query", e); + } + } , RESCORE_QUERY_FIELD); + QUERY_RESCORE_PARSER.declareFloat(InnerBuilder::setQueryWeight, QUERY_WEIGHT_FIELD); + QUERY_RESCORE_PARSER.declareFloat(InnerBuilder::setRescoreQueryWeight, RESCORE_QUERY_WEIGHT_FIELD); + QUERY_RESCORE_PARSER.declareString((struct, value) -> struct.setScoreMode(QueryRescoreMode.fromString(value)), SCORE_MODE_FIELD); + } + + /** + * Creates a new {@link QueryRescorerBuilder} instance + * @param builder the query builder to build the rescore query from + */ + public QueryRescorerBuilder(QueryBuilder builder) { + this.queryBuilder = builder; + } + + /** + * @return the query used for this rescore query + */ + public QueryBuilder getRescoreQuery() { + return this.queryBuilder; + } + + /** + * Sets the original query weight for rescoring. The default is 1.0 + */ + public QueryRescorerBuilder setQueryWeight(float queryWeight) { + this.queryWeight = queryWeight; + return this; + } + + + /** + * Gets the original query weight for rescoring. The default is 1.0 + */ + public float getQueryWeight() { + return this.queryWeight; + } + + /** + * Sets the original query weight for rescoring. The default is 1.0 + */ + public QueryRescorerBuilder setRescoreQueryWeight(float rescoreQueryWeight) { + this.rescoreQueryWeight = rescoreQueryWeight; + return this; + } + + /** + * Gets the original query weight for rescoring. The default is 1.0 + */ + public float getRescoreQueryWeight() { + return this.rescoreQueryWeight; + } + + /** + * Sets the original query score mode. The default is {@link QueryRescoreMode#Total}. + */ + public QueryRescorerBuilder setScoreMode(QueryRescoreMode scoreMode) { + this.scoreMode = scoreMode; + return this; + } + + /** + * Gets the original query score mode. The default is total + */ + public QueryRescoreMode getScoreMode() { + return this.scoreMode; + } + + @Override + public void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.field(RESCORE_QUERY_FIELD.getPreferredName(), queryBuilder); + builder.field(QUERY_WEIGHT_FIELD.getPreferredName(), queryWeight); + builder.field(RESCORE_QUERY_WEIGHT_FIELD.getPreferredName(), rescoreQueryWeight); + builder.field(SCORE_MODE_FIELD.getPreferredName(), scoreMode.name().toLowerCase(Locale.ROOT)); + builder.endObject(); + } + + public QueryRescorerBuilder fromXContent(QueryParseContext parseContext) throws IOException { + InnerBuilder innerBuilder = QUERY_RESCORE_PARSER.parse(parseContext.parser(), new InnerBuilder(), parseContext); + return innerBuilder.build(); + } + + @Override + public QueryRescoreContext build(QueryShardContext context) throws IOException { + org.elasticsearch.search.rescore.QueryRescorer rescorer = new org.elasticsearch.search.rescore.QueryRescorer(); + QueryRescoreContext queryRescoreContext = new QueryRescoreContext(rescorer); + queryRescoreContext.setQuery(this.queryBuilder.toQuery(context)); + queryRescoreContext.setQueryWeight(this.queryWeight); + queryRescoreContext.setRescoreQueryWeight(this.rescoreQueryWeight); + queryRescoreContext.setScoreMode(this.scoreMode); + if (this.windowSize != null) { + queryRescoreContext.setWindowSize(this.windowSize); + } + return queryRescoreContext; + } + + @Override + public final int hashCode() { + int result = super.hashCode(); + return 31 * result + Objects.hash(scoreMode, queryWeight, rescoreQueryWeight, queryBuilder); + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + QueryRescorerBuilder other = (QueryRescorerBuilder) obj; + return super.equals(obj) && + Objects.equals(scoreMode, other.scoreMode) && + Objects.equals(queryWeight, other.queryWeight) && + Objects.equals(rescoreQueryWeight, other.rescoreQueryWeight) && + Objects.equals(queryBuilder, other.queryBuilder); + } + + @Override + public QueryRescorerBuilder doReadFrom(StreamInput in) throws IOException { + QueryRescorerBuilder rescorer = new QueryRescorerBuilder(in.readQuery()); + rescorer.setScoreMode(QueryRescoreMode.PROTOTYPE.readFrom(in)); + rescorer.setRescoreQueryWeight(in.readFloat()); + rescorer.setQueryWeight(in.readFloat()); + return rescorer; + } + + @Override + public void doWriteTo(StreamOutput out) throws IOException { + out.writeQuery(queryBuilder); + scoreMode.writeTo(out); + out.writeFloat(rescoreQueryWeight); + out.writeFloat(queryWeight); + } + + @Override + public String getWriteableName() { + return NAME; + } + + /** + * Helper to be able to use {@link ObjectParser}, since we need the inner query builder + * for the constructor of {@link QueryRescorerBuilder}, but {@link ObjectParser} only + * allows filling properties of an already constructed value. + */ + private class InnerBuilder { + + private QueryBuilder queryBuilder; + private float rescoreQueryWeight = DEFAULT_RESCORE_QUERYWEIGHT; + private float queryWeight = DEFAULT_QUERYWEIGHT; + private QueryRescoreMode scoreMode = DEFAULT_SCORE_MODE; + + void setQueryBuilder(QueryBuilder builder) { + this.queryBuilder = builder; + } + + QueryRescorerBuilder build() { + QueryRescorerBuilder queryRescoreBuilder = new QueryRescorerBuilder(queryBuilder); + queryRescoreBuilder.setQueryWeight(queryWeight); + queryRescoreBuilder.setRescoreQueryWeight(rescoreQueryWeight); + queryRescoreBuilder.setScoreMode(scoreMode); + return queryRescoreBuilder; + } + + void setQueryWeight(float queryWeight) { + this.queryWeight = queryWeight; + } + + void setRescoreQueryWeight(float rescoreQueryWeight) { + this.rescoreQueryWeight = rescoreQueryWeight; + } + + void setScoreMode(QueryRescoreMode scoreMode) { + this.scoreMode = scoreMode; + } + } +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java b/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java index 7510d24f82d..8dad07a5430 100644 --- a/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java @@ -20,255 +20,140 @@ package org.elasticsearch.search.rescore; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.rescore.QueryRescorer.QueryRescoreContext; import java.io.IOException; -import java.util.Locale; import java.util.Objects; -public class RescoreBuilder implements ToXContent, Writeable { +/** + * The abstract base builder for instances of {@link RescoreBuilder}. + */ +public abstract class RescoreBuilder> implements ToXContent, NamedWriteable { - private Rescorer rescorer; - private Integer windowSize; - public static final RescoreBuilder PROTOYPE = new RescoreBuilder(new QueryRescorer(new MatchAllQueryBuilder())); + protected Integer windowSize; - public RescoreBuilder(Rescorer rescorer) { - if (rescorer == null) { - throw new IllegalArgumentException("rescorer cannot be null"); - } - this.rescorer = rescorer; - } + private static ParseField WINDOW_SIZE_FIELD = new ParseField("window_size"); - public Rescorer rescorer() { - return this.rescorer; - } - - public RescoreBuilder windowSize(int windowSize) { + @SuppressWarnings("unchecked") + public RB windowSize(int windowSize) { this.windowSize = windowSize; - return this; + return (RB) this; } public Integer windowSize() { return windowSize; } + public static RescoreBuilder parseFromXContent(QueryParseContext parseContext) throws IOException { + XContentParser parser = parseContext.parser(); + String fieldName = null; + RescoreBuilder rescorer = null; + Integer windowSize = null; + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + } else if (token.isValue()) { + if (parseContext.parseFieldMatcher().match(fieldName, WINDOW_SIZE_FIELD)) { + windowSize = parser.intValue(); + } else { + throw new ParsingException(parser.getTokenLocation(), "rescore doesn't support [" + fieldName + "]"); + } + } else if (token == XContentParser.Token.START_OBJECT) { + // we only have QueryRescorer at this point + if (QueryRescorerBuilder.NAME.equals(fieldName)) { + rescorer = QueryRescorerBuilder.PROTOTYPE.fromXContent(parseContext); + } else { + throw new ParsingException(parser.getTokenLocation(), "rescore doesn't support rescorer with name [" + fieldName + "]"); + } + } else { + throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "] after [" + fieldName + "]"); + } + } + if (rescorer == null) { + throw new ParsingException(parser.getTokenLocation(), "missing rescore type"); + } + if (windowSize != null) { + rescorer.windowSize(windowSize.intValue()); + } + return rescorer; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); if (windowSize != null) { builder.field("window_size", windowSize); } - rescorer.toXContent(builder, params); + doXContent(builder, params); + builder.endObject(); return builder; } - public static QueryRescorer queryRescorer(QueryBuilder queryBuilder) { - return new QueryRescorer(queryBuilder); + protected abstract void doXContent(XContentBuilder builder, Params params) throws IOException; + + public abstract QueryRescoreContext build(QueryShardContext context) throws IOException; + + public static QueryRescorerBuilder queryRescorer(QueryBuilder queryBuilder) { + return new QueryRescorerBuilder(queryBuilder); } @Override - public final int hashCode() { - return Objects.hash(windowSize, rescorer); + public int hashCode() { + return Objects.hash(windowSize); } @Override - public final boolean equals(Object obj) { + public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } + @SuppressWarnings("rawtypes") RescoreBuilder other = (RescoreBuilder) obj; - return Objects.equals(windowSize, other.windowSize) && - Objects.equals(rescorer, other.rescorer); + return Objects.equals(windowSize, other.windowSize); } @Override - public RescoreBuilder readFrom(StreamInput in) throws IOException { - RescoreBuilder builder = new RescoreBuilder(in.readRescorer()); - Integer windowSize = in.readOptionalVInt(); - if (windowSize != null) { - builder.windowSize(windowSize); - } + public RB readFrom(StreamInput in) throws IOException { + RB builder = doReadFrom(in); + builder.windowSize = in.readOptionalVInt(); return builder; } + protected abstract RB doReadFrom(StreamInput in) throws IOException; + @Override public void writeTo(StreamOutput out) throws IOException { - out.writeRescorer(rescorer); + doWriteTo(out); out.writeOptionalVInt(this.windowSize); } + protected abstract void doWriteTo(StreamOutput out) throws IOException; + @Override public final String toString() { try { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.prettyPrint(); - builder.startObject(); toXContent(builder, EMPTY_PARAMS); - builder.endObject(); return builder.string(); } catch (Exception e) { return "{ \"error\" : \"" + ExceptionsHelper.detailedMessage(e) + "\"}"; } } - - public static abstract class Rescorer implements ToXContent, NamedWriteable { - - private String name; - - public Rescorer(String name) { - this.name = name; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(name); - builder = innerToXContent(builder, params); - builder.endObject(); - return builder; - } - - protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException; - - @Override - public abstract int hashCode(); - - @Override - public abstract boolean equals(Object obj); - } - - public static class QueryRescorer extends Rescorer { - - private static final String NAME = "query"; - public static final QueryRescorer PROTOTYPE = new QueryRescorer(new MatchAllQueryBuilder()); - public static final float DEFAULT_RESCORE_QUERYWEIGHT = 1.0f; - public static final float DEFAULT_QUERYWEIGHT = 1.0f; - public static final QueryRescoreMode DEFAULT_SCORE_MODE = QueryRescoreMode.Total; - private final QueryBuilder queryBuilder; - private float rescoreQueryWeight = DEFAULT_RESCORE_QUERYWEIGHT; - private float queryWeight = DEFAULT_QUERYWEIGHT; - private QueryRescoreMode scoreMode = DEFAULT_SCORE_MODE; - - /** - * Creates a new {@link QueryRescorer} instance - * @param builder the query builder to build the rescore query from - */ - public QueryRescorer(QueryBuilder builder) { - super(NAME); - this.queryBuilder = builder; - } - - /** - * @return the query used for this rescore query - */ - public QueryBuilder getRescoreQuery() { - return this.queryBuilder; - } - - /** - * Sets the original query weight for rescoring. The default is 1.0 - */ - public QueryRescorer setQueryWeight(float queryWeight) { - this.queryWeight = queryWeight; - return this; - } - - - /** - * Gets the original query weight for rescoring. The default is 1.0 - */ - public float getQueryWeight() { - return this.queryWeight; - } - - /** - * Sets the original query weight for rescoring. The default is 1.0 - */ - public QueryRescorer setRescoreQueryWeight(float rescoreQueryWeight) { - this.rescoreQueryWeight = rescoreQueryWeight; - return this; - } - - /** - * Gets the original query weight for rescoring. The default is 1.0 - */ - public float getRescoreQueryWeight() { - return this.rescoreQueryWeight; - } - - /** - * Sets the original query score mode. The default is {@link QueryRescoreMode#Total}. - */ - public QueryRescorer setScoreMode(QueryRescoreMode scoreMode) { - this.scoreMode = scoreMode; - return this; - } - - /** - * Gets the original query score mode. The default is total - */ - public QueryRescoreMode getScoreMode() { - return this.scoreMode; - } - - @Override - protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("rescore_query", queryBuilder); - builder.field("query_weight", queryWeight); - builder.field("rescore_query_weight", rescoreQueryWeight); - builder.field("score_mode", scoreMode.name().toLowerCase(Locale.ROOT)); - return builder; - } - - @Override - public final int hashCode() { - return Objects.hash(getClass(), scoreMode, queryWeight, rescoreQueryWeight, queryBuilder); - } - - @Override - public final boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - QueryRescorer other = (QueryRescorer) obj; - return Objects.equals(scoreMode, other.scoreMode) && - Objects.equals(queryWeight, other.queryWeight) && - Objects.equals(rescoreQueryWeight, other.rescoreQueryWeight) && - Objects.equals(queryBuilder, other.queryBuilder); - } - - @Override - public QueryRescorer readFrom(StreamInput in) throws IOException { - QueryRescorer rescorer = new QueryRescorer(in.readQuery()); - rescorer.setScoreMode(QueryRescoreMode.PROTOTYPE.readFrom(in)); - rescorer.setRescoreQueryWeight(in.readFloat()); - rescorer.setQueryWeight(in.readFloat()); - return rescorer; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeQuery(queryBuilder); - scoreMode.writeTo(out); - out.writeFloat(rescoreQueryWeight); - out.writeFloat(queryWeight); - } - - @Override - public String getWriteableName() { - return NAME; - } - } } diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescoreParseElement.java b/core/src/main/java/org/elasticsearch/search/rescore/RescoreParseElement.java index 7f9f2725fbc..149db6cec2c 100644 --- a/core/src/main/java/org/elasticsearch/search/rescore/RescoreParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/rescore/RescoreParseElement.java @@ -21,9 +21,12 @@ package org.elasticsearch.search.rescore; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.internal.SearchContext; +import java.io.IOException; + /** * */ @@ -33,14 +36,14 @@ public class RescoreParseElement implements SearchParseElement { public void parse(XContentParser parser, SearchContext context) throws Exception { if (parser.currentToken() == XContentParser.Token.START_ARRAY) { while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - parseSingleRescoreContext(parser, context); + context.addRescore(parseSingleRescoreContext(parser, context.indexShard().getQueryShardContext())); } } else { - parseSingleRescoreContext(parser, context); + context.addRescore(parseSingleRescoreContext(parser, context.indexShard().getQueryShardContext())); } } - public void parseSingleRescoreContext(XContentParser parser, SearchContext context) throws Exception { + public RescoreSearchContext parseSingleRescoreContext(XContentParser parser, QueryShardContext context) throws ElasticsearchParseException, IOException { String fieldName = null; RescoreSearchContext rescoreContext = null; Integer windowSize = null; @@ -71,7 +74,7 @@ public class RescoreParseElement implements SearchParseElement { if (windowSize != null) { rescoreContext.setWindowSize(windowSize.intValue()); } - context.addRescore(rescoreContext); + return rescoreContext; } } diff --git a/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java b/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java index 3c90289fde5..e3465a4df8f 100644 --- a/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java +++ b/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.TopDocs; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -68,11 +69,11 @@ public interface Rescorer { * Parses the {@link RescoreSearchContext} for this impelementation * * @param parser the parser to read the context from - * @param context the current search context + * @param context the current shard context * @return the parsed {@link RescoreSearchContext} * @throws IOException if an {@link IOException} occurs while parsing the context */ - public RescoreSearchContext parse(XContentParser parser, SearchContext context) throws IOException; + public RescoreSearchContext parse(XContentParser parser, QueryShardContext context) throws IOException; /** * Extracts all terms needed to exectue this {@link Rescorer}. This method @@ -81,7 +82,7 @@ public interface Rescorer { * {@link SearchType#DFS_QUERY_THEN_FETCH} */ public void extractTerms(SearchContext context, RescoreSearchContext rescoreContext, Set termsSet); - + /* * TODO: At this point we only have one implemenation which modifies the * TopDocs given. Future implemenations might return actual resutls that diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 62f3e22de3d..0c317271248 100644 --- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -19,11 +19,6 @@ package org.elasticsearch.search.builder; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; - import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -58,7 +53,7 @@ import org.elasticsearch.search.fetch.innerhits.InnerHitsBuilder; import org.elasticsearch.search.fetch.innerhits.InnerHitsBuilder.InnerHit; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilderTests; -import org.elasticsearch.search.rescore.RescoreBuilder; +import org.elasticsearch.search.rescore.QueryRescoreBuilderTests; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.suggest.SuggestBuilder; @@ -69,6 +64,11 @@ import org.elasticsearch.threadpool.ThreadPoolModule; import org.junit.AfterClass; import org.junit.BeforeClass; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + import static org.hamcrest.Matchers.equalTo; public class SearchSourceBuilderTests extends ESTestCase { @@ -281,10 +281,7 @@ public class SearchSourceBuilderTests extends ESTestCase { if (randomBoolean()) { int numRescores = randomIntBetween(1, 5); for (int i = 0; i < numRescores; i++) { - // NORELEASE need a random rescore builder method - RescoreBuilder rescoreBuilder = new RescoreBuilder(RescoreBuilder.queryRescorer(QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), - randomAsciiOfLengthBetween(5, 20)))); - builder.addRescorer(rescoreBuilder); + builder.addRescorer(QueryRescoreBuilderTests.randomRescoreBuilder()); } } if (randomBoolean()) { diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java index 5644f893603..07f51772a7d 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java @@ -37,9 +37,9 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.rescore.QueryRescoreMode; import org.elasticsearch.search.rescore.RescoreBuilder; -import org.elasticsearch.search.rescore.RescoreBuilder.QueryRescorer; +import org.elasticsearch.search.rescore.QueryRescoreMode; +import org.elasticsearch.search.rescore.QueryRescorerBuilder; import org.elasticsearch.test.ESIntegTestCase; import java.util.Arrays; @@ -538,7 +538,7 @@ public class QueryRescorerIT extends ESIntegTestCase { String[] scoreModes = new String[]{ "max", "min", "avg", "total", "multiply", "" }; String[] descriptionModes = new String[]{ "max of:", "min of:", "avg of:", "sum of:", "product of:", "sum of:" }; for (int innerMode = 0; innerMode < scoreModes.length; innerMode++) { - QueryRescorer innerRescoreQuery = RescoreBuilder.queryRescorer(QueryBuilders.matchQuery("field1", "the quick brown").boost(4.0f)) + QueryRescorerBuilder innerRescoreQuery = RescoreBuilder.queryRescorer(QueryBuilders.matchQuery("field1", "the quick brown").boost(4.0f)) .setQueryWeight(0.5f).setRescoreQueryWeight(0.4f); if (!"".equals(scoreModes[innerMode])) { @@ -561,7 +561,7 @@ public class QueryRescorerIT extends ESIntegTestCase { } for (int outerMode = 0; outerMode < scoreModes.length; outerMode++) { - QueryRescorer outerRescoreQuery = RescoreBuilder.queryRescorer(QueryBuilders.matchQuery("field1", "the quick brown") + QueryRescorerBuilder outerRescoreQuery = RescoreBuilder.queryRescorer(QueryBuilders.matchQuery("field1", "the quick brown") .boost(4.0f)).setQueryWeight(0.5f).setRescoreQueryWeight(0.4f); if (!"".equals(scoreModes[outerMode])) { @@ -572,7 +572,7 @@ public class QueryRescorerIT extends ESIntegTestCase { .prepareSearch() .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) .setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR)) - .addRescorer(innerRescoreQuery, 5).addRescorer(outerRescoreQuery, 10) + .addRescorer(innerRescoreQuery, 5).addRescorer(outerRescoreQuery.windowSize(10)) .setExplain(true).get(); assertHitCount(searchResponse, 3); assertFirstHit(searchResponse, hasId("1")); @@ -599,7 +599,7 @@ public class QueryRescorerIT extends ESIntegTestCase { for (int i = 0; i < numDocs - 4; i++) { String[] intToEnglish = new String[] { English.intToEnglish(i), English.intToEnglish(i + 1), English.intToEnglish(i + 2), English.intToEnglish(i + 3) }; - QueryRescorer rescoreQuery = RescoreBuilder + QueryRescorerBuilder rescoreQuery = RescoreBuilder .queryRescorer( QueryBuilders.boolQuery() .disableCoord(true) @@ -682,10 +682,10 @@ public class QueryRescorerIT extends ESIntegTestCase { public void testMultipleRescores() throws Exception { int numDocs = indexRandomNumbers("keyword", 1, true); - QueryRescorer eightIsGreat = RescoreBuilder.queryRescorer( + QueryRescorerBuilder eightIsGreat = RescoreBuilder.queryRescorer( QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("field1", English.intToEnglish(8)), ScoreFunctionBuilders.weightFactorFunction(1000.0f)).boostMode(CombineFunction.REPLACE)).setScoreMode(QueryRescoreMode.Total); - QueryRescorer sevenIsBetter = RescoreBuilder.queryRescorer( + QueryRescorerBuilder sevenIsBetter = RescoreBuilder.queryRescorer( QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("field1", English.intToEnglish(7)), ScoreFunctionBuilders.weightFactorFunction(10000.0f)).boostMode(CombineFunction.REPLACE)) .setScoreMode(QueryRescoreMode.Total); @@ -703,10 +703,10 @@ public class QueryRescorerIT extends ESIntegTestCase { // We have no idea what the second hit will be because we didn't get a chance to look for seven // Now use one rescore to drag the number we're looking for into the window of another - QueryRescorer ninetyIsGood = RescoreBuilder.queryRescorer( + QueryRescorerBuilder ninetyIsGood = RescoreBuilder.queryRescorer( QueryBuilders.functionScoreQuery(QueryBuilders.queryStringQuery("*ninety*"), ScoreFunctionBuilders.weightFactorFunction(1000.0f)) .boostMode(CombineFunction.REPLACE)).setScoreMode(QueryRescoreMode.Total); - QueryRescorer oneToo = RescoreBuilder.queryRescorer( + QueryRescorerBuilder oneToo = RescoreBuilder.queryRescorer( QueryBuilders.functionScoreQuery(QueryBuilders.queryStringQuery("*one*"), ScoreFunctionBuilders.weightFactorFunction(1000.0f)) .boostMode(CombineFunction.REPLACE)).setScoreMode(QueryRescoreMode.Total); request.clearRescorers().addRescorer(ninetyIsGood, numDocs).addRescorer(oneToo, 10); diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java index 383bde00b01..7446f99f571 100644 --- a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java @@ -19,13 +19,6 @@ package org.elasticsearch.search.highlight; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParseFieldMatcher; @@ -64,6 +57,13 @@ import org.elasticsearch.test.IndexSettingsModule; import org.junit.AfterClass; import org.junit.BeforeClass; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java index 2aa55f8b626..1305aa727f5 100644 --- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java @@ -19,15 +19,38 @@ package org.elasticsearch.search.rescore; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperBuilders; +import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.rescore.RescoreBuilder.QueryRescorer; -import org.elasticsearch.search.rescore.RescoreBuilder.Rescorer; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.rescore.QueryRescorer.QueryRescoreContext; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -40,6 +63,7 @@ public class QueryRescoreBuilderTests extends ESTestCase { private static final int NUMBER_OF_TESTBUILDERS = 20; private static NamedWriteableRegistry namedWriteableRegistry; + private static IndicesQueriesRegistry indicesQueriesRegistry; /** * setup for the whole base test class @@ -47,13 +71,14 @@ public class QueryRescoreBuilderTests extends ESTestCase { @BeforeClass public static void init() { namedWriteableRegistry = new NamedWriteableRegistry(); - namedWriteableRegistry.registerPrototype(Rescorer.class, org.elasticsearch.search.rescore.RescoreBuilder.QueryRescorer.PROTOTYPE); - namedWriteableRegistry.registerPrototype(QueryBuilder.class, new MatchAllQueryBuilder()); + namedWriteableRegistry.registerPrototype(RescoreBuilder.class, QueryRescorerBuilder.PROTOTYPE); + indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).buildQueryParserRegistry(); } @AfterClass public static void afterClass() throws Exception { namedWriteableRegistry = null; + indicesQueriesRegistry = null; } /** @@ -61,8 +86,8 @@ public class QueryRescoreBuilderTests extends ESTestCase { */ public void testSerialization() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { - RescoreBuilder original = randomRescoreBuilder(); - RescoreBuilder deserialized = serializedCopy(original); + RescoreBuilder original = randomRescoreBuilder(); + RescoreBuilder deserialized = serializedCopy(original); assertEquals(deserialized, original); assertEquals(deserialized.hashCode(), original.hashCode()); assertNotSame(deserialized, original); @@ -74,7 +99,7 @@ public class QueryRescoreBuilderTests extends ESTestCase { */ public void testEqualsAndHashcode() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { - RescoreBuilder firstBuilder = randomRescoreBuilder(); + RescoreBuilder firstBuilder = randomRescoreBuilder(); assertFalse("rescore builder is equal to null", firstBuilder.equals(null)); assertFalse("rescore builder is equal to incompatible type", firstBuilder.equals("")); assertTrue("rescore builder is not equal to self", firstBuilder.equals(firstBuilder)); @@ -82,13 +107,13 @@ public class QueryRescoreBuilderTests extends ESTestCase { equalTo(firstBuilder.hashCode())); assertThat("different rescore builder should not be equal", mutate(firstBuilder), not(equalTo(firstBuilder))); - RescoreBuilder secondBuilder = serializedCopy(firstBuilder); + RescoreBuilder secondBuilder = serializedCopy(firstBuilder); assertTrue("rescore builder is not equal to self", secondBuilder.equals(secondBuilder)); assertTrue("rescore builder is not equal to its copy", firstBuilder.equals(secondBuilder)); assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder)); assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(firstBuilder.hashCode())); - RescoreBuilder thirdBuilder = serializedCopy(secondBuilder); + RescoreBuilder thirdBuilder = serializedCopy(secondBuilder); assertTrue("rescore builder is not equal to self", thirdBuilder.equals(thirdBuilder)); assertTrue("rescore builder is not equal to its copy", secondBuilder.equals(thirdBuilder)); assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(thirdBuilder.hashCode())); @@ -99,8 +124,162 @@ public class QueryRescoreBuilderTests extends ESTestCase { } } - private RescoreBuilder mutate(RescoreBuilder original) throws IOException { - RescoreBuilder mutation = serializedCopy(original); + /** + * creates random rescorer, renders it to xContent and back to new instance that should be equal to original + */ + public void testFromXContent() throws IOException { + QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); + context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY)); + for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { + RescoreBuilder rescoreBuilder = randomRescoreBuilder(); + + XContentParser parser = createParser(rescoreBuilder); + context.reset(parser); + parser.nextToken(); + RescoreBuilder secondRescoreBuilder = RescoreBuilder.parseFromXContent(context); + assertNotSame(rescoreBuilder, secondRescoreBuilder); + assertEquals(rescoreBuilder, secondRescoreBuilder); + assertEquals(rescoreBuilder.hashCode(), secondRescoreBuilder.hashCode()); + } + } + + private static XContentParser createParser(RescoreBuilder rescoreBuilder) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + rescoreBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); + return XContentHelper.createParser(builder.bytes()); + } + + /** + * test that build() outputs a {@link RescoreSearchContext} that is similar to the one + * we would get when parsing the xContent the test rescore builder is rendering out + */ + public void testBuildRescoreSearchContext() throws ElasticsearchParseException, IOException { + Settings indexSettings = Settings.settingsBuilder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + Index index = new Index(randomAsciiOfLengthBetween(1, 10)); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings); + // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer + QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, null, indicesQueriesRegistry) { + @Override + public MappedFieldType fieldMapper(String name) { + StringFieldMapper.Builder builder = MapperBuilders.stringField(name); + return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType(); + } + }; + + for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { + RescoreBuilder rescoreBuilder = randomRescoreBuilder(); + QueryRescoreContext rescoreContext = (QueryRescoreContext) rescoreBuilder.build(mockShardContext); + XContentParser parser = createParser(rescoreBuilder); + + QueryRescoreContext parsedRescoreContext = (QueryRescoreContext) new RescoreParseElement().parseSingleRescoreContext(parser, mockShardContext); + assertNotSame(rescoreContext, parsedRescoreContext); + assertEquals(rescoreContext.window(), parsedRescoreContext.window()); + assertEquals(rescoreContext.query(), parsedRescoreContext.query()); + assertEquals(rescoreContext.queryWeight(), parsedRescoreContext.queryWeight(), Float.MIN_VALUE); + assertEquals(rescoreContext.rescoreQueryWeight(), parsedRescoreContext.rescoreQueryWeight(), Float.MIN_VALUE); + assertEquals(rescoreContext.scoreMode(), parsedRescoreContext.scoreMode()); + } + } + + /** + * test parsing exceptions for incorrect rescorer syntax + */ + public void testUnknownFieldsExpection() throws IOException { + QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); + context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY)); + + String rescoreElement = "{\n" + + " \"window_size\" : 20,\n" + + " \"bad_rescorer_name\" : { }\n" + + "}\n"; + prepareContext(context, rescoreElement); + try { + RescoreBuilder.parseFromXContent(context); + fail("expected a parsing exception"); + } catch (ParsingException e) { + assertEquals("rescore doesn't support rescorer with name [bad_rescorer_name]", e.getMessage()); + } + + rescoreElement = "{\n" + + " \"bad_fieldName\" : 20\n" + + "}\n"; + prepareContext(context, rescoreElement); + try { + RescoreBuilder.parseFromXContent(context); + fail("expected a parsing exception"); + } catch (ParsingException e) { + assertEquals("rescore doesn't support [bad_fieldName]", e.getMessage()); + } + + rescoreElement = "{\n" + + " \"window_size\" : 20,\n" + + " \"query\" : [ ]\n" + + "}\n"; + prepareContext(context, rescoreElement); + try { + RescoreBuilder.parseFromXContent(context); + fail("expected a parsing exception"); + } catch (ParsingException e) { + assertEquals("unexpected token [START_ARRAY] after [query]", e.getMessage()); + } + + rescoreElement = "{ }"; + prepareContext(context, rescoreElement); + try { + RescoreBuilder.parseFromXContent(context); + fail("expected a parsing exception"); + } catch (ParsingException e) { + assertEquals("missing rescore type", e.getMessage()); + } + + rescoreElement = "{\n" + + " \"window_size\" : 20,\n" + + " \"query\" : { \"bad_fieldname\" : 1.0 } \n" + + "}\n"; + prepareContext(context, rescoreElement); + try { + RescoreBuilder.parseFromXContent(context); + fail("expected a parsing exception"); + } catch (IllegalArgumentException e) { + assertEquals("[query] unknown field [bad_fieldname], parser not found", e.getMessage()); + } + + rescoreElement = "{\n" + + " \"window_size\" : 20,\n" + + " \"query\" : { \"rescore_query\" : { \"unknown_queryname\" : { } } } \n" + + "}\n"; + prepareContext(context, rescoreElement); + try { + RescoreBuilder.parseFromXContent(context); + fail("expected a parsing exception"); + } catch (ParsingException e) { + assertEquals("[query] failed to parse field [rescore_query]", e.getMessage()); + } + + rescoreElement = "{\n" + + " \"window_size\" : 20,\n" + + " \"query\" : { \"rescore_query\" : { \"match_all\" : { } } } \n" + + "}\n"; + prepareContext(context, rescoreElement); + RescoreBuilder.parseFromXContent(context); + } + + /** + * create a new parser from the rescorer string representation and reset context with it + */ + private static void prepareContext(QueryParseContext context, String rescoreElement) throws IOException { + XContentParser parser = XContentFactory.xContent(rescoreElement).createParser(rescoreElement); + context.reset(parser); + // move to first token, this is where the internal fromXContent + assertTrue(parser.nextToken() == XContentParser.Token.START_OBJECT); + } + + private static RescoreBuilder mutate(RescoreBuilder original) throws IOException { + RescoreBuilder mutation = serializedCopy(original); if (randomBoolean()) { Integer windowSize = original.windowSize(); if (windowSize != null) { @@ -109,7 +288,7 @@ public class QueryRescoreBuilderTests extends ESTestCase { mutation.windowSize(randomIntBetween(0, 100)); } } else { - QueryRescorer queryRescorer = (QueryRescorer) mutation.rescorer(); + QueryRescorerBuilder queryRescorer = (QueryRescorerBuilder) mutation; switch (randomIntBetween(0, 3)) { case 0: queryRescorer.setQueryWeight(queryRescorer.getQueryWeight() + 0.1f); @@ -138,10 +317,10 @@ public class QueryRescoreBuilderTests extends ESTestCase { /** * create random shape that is put under test */ - private static RescoreBuilder randomRescoreBuilder() { + public static org.elasticsearch.search.rescore.QueryRescorerBuilder randomRescoreBuilder() { QueryBuilder queryBuilder = new MatchAllQueryBuilder().boost(randomFloat()).queryName(randomAsciiOfLength(20)); - org.elasticsearch.search.rescore.RescoreBuilder.QueryRescorer rescorer = new - org.elasticsearch.search.rescore.RescoreBuilder.QueryRescorer(queryBuilder); + org.elasticsearch.search.rescore.QueryRescorerBuilder rescorer = new + org.elasticsearch.search.rescore.QueryRescorerBuilder(queryBuilder); if (randomBoolean()) { rescorer.setQueryWeight(randomFloat()); } @@ -151,18 +330,17 @@ public class QueryRescoreBuilderTests extends ESTestCase { if (randomBoolean()) { rescorer.setScoreMode(randomFrom(QueryRescoreMode.values())); } - RescoreBuilder builder = new RescoreBuilder(rescorer); if (randomBoolean()) { - builder.windowSize(randomIntBetween(0, 100)); + rescorer.windowSize(randomIntBetween(0, 100)); } - return builder; + return rescorer; } - private static RescoreBuilder serializedCopy(RescoreBuilder original) throws IOException { + private static RescoreBuilder serializedCopy(RescoreBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { - original.writeTo(output); + output.writeRescorer(original); try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { - return RescoreBuilder.PROTOYPE.readFrom(in); + return in.readRescorer(); } } } diff --git a/docs/plugins/ingest-geoip.asciidoc b/docs/plugins/ingest-geoip.asciidoc index 539c29971a4..2c0663b5958 100644 --- a/docs/plugins/ingest-geoip.asciidoc +++ b/docs/plugins/ingest-geoip.asciidoc @@ -4,7 +4,7 @@ The GeoIP processor adds information about the geographical location of IP addresses, based on data from the Maxmind databases. This processor adds this information by default under the `geoip` field. -The ingest plugin ships by default with the GeoLite2 City and GeoLite2 Country geoip2 databases from Maxmind made available +The ingest-geoip plugin ships by default with the GeoLite2 City and GeoLite2 Country geoip2 databases from Maxmind made available under the CCA-ShareAlike 3.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/ The GeoIP processor can run with other geoip2 databases from Maxmind. The files must be copied into the geoip config directory @@ -18,7 +18,7 @@ is located at `$ES_HOME/config/ingest/geoip` and holds the shipped databases too | Name | Required | Default | Description | `source_field` | yes | - | The field to get the ip address or hostname from for the geographical lookup. | `target_field` | no | geoip | The field that will hold the geographical information looked up from the Maxmind database. -| `database_file` | no | GeoLite2-City.mmdb | The database filename in the geoip config directory. The ingest plugin ships with the GeoLite2-City.mmdb and GeoLite2-Country.mmdb files. +| `database_file` | no | GeoLite2-City.mmdb | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb and GeoLite2-Country.mmdb files. | `fields` | no | [`continent_name`, `country_iso_code`, `region_name`, `city_name`, `location`] <1> | Controls what properties are added to the `target_field` based on the geoip lookup. |====== diff --git a/docs/reference/ingest/ingest.asciidoc b/docs/reference/ingest/ingest.asciidoc index 0c049f82b69..ee724d4ad07 100644 --- a/docs/reference/ingest/ingest.asciidoc +++ b/docs/reference/ingest/ingest.asciidoc @@ -1,23 +1,22 @@ [[ingest]] -== Ingest Plugin +== Ingest Node -The ingest plugin can be used to pre-process documents before the actual indexing takes place. -This pre-processing happens by the ingest plugin that intercepts bulk and index requests, applies the +Ingest node can be used to pre-process documents before the actual indexing takes place. +This pre-processing happens by an ingest node that intercepts bulk and index requests, applies the transformations and then passes the documents back to the index or bulk APIs. -The ingest plugin is disabled by default. In order to enable the ingest plugin the following +Ingest node is enabled by default. In order to disable ingest the following setting should be configured in the elasticsearch.yml file: [source,yaml] -------------------------------------------------- -node.ingest: true +node.ingest: false -------------------------------------------------- -The ingest plugin can be installed and enabled on any node. It is possible to run ingest -on an master and or data node or have dedicated client nodes that run with ingest. +It is possible to enable ingest on any node or have dedicated ingest nodes. In order to pre-process document before indexing the `pipeline` parameter should be used -on an index or bulk request to tell the ingest plugin what pipeline is going to be used. +on an index or bulk request to tell Ingest what pipeline is going to be used. [source,js] -------------------------------------------------- @@ -606,9 +605,9 @@ The following example sets the id of a document to `1`: The following metadata fields are accessible by a processor: `_index`, `_type`, `_id`, `_routing`, `_parent`, `_timestamp` and `_ttl`. -Beyond metadata fields and source fields, the ingest plugin also adds ingest metadata to documents being processed. -These metadata properties are accessible under the `_ingest` key. Currently the ingest plugin adds the ingest timestamp -under `_ingest.timestamp` key to the ingest metadata, which is the time the ingest plugin received the index or bulk +Beyond metadata fields and source fields, ingest also adds ingest metadata to documents being processed. +These metadata properties are accessible under the `_ingest` key. Currently ingest adds the ingest timestamp +under `_ingest.timestamp` key to the ingest metadata, which is the time ES received the index or bulk request to pre-process. But any processor is free to add more ingest related metadata to it. Ingest metadata is transient and is lost after a document has been processed by the pipeline and thus ingest metadata won't be indexed. @@ -626,7 +625,7 @@ The following example adds a field with the name `received` and the value is the As opposed to Elasticsearch metadata fields, the ingest metadata field name _ingest can be used as a valid field name in the source of a document. Use _source._ingest to refer to it, otherwise _ingest will be interpreted as ingest -metadata fields by the ingest plugin. +metadata fields. A number of processor settings also support templating. Settings that support templating can have zero or more template snippets. A template snippet begins with `{{` and ends with `}}`. diff --git a/docs/reference/migration/migrate_3_0.asciidoc b/docs/reference/migration/migrate_3_0.asciidoc index b4aa2d6d844..699805ac7e1 100644 --- a/docs/reference/migration/migrate_3_0.asciidoc +++ b/docs/reference/migration/migrate_3_0.asciidoc @@ -545,6 +545,10 @@ to index a document only if it doesn't already exist. `InternalLineStringBuilder` is removed in favour of `LineStringBuilder`, `InternalPolygonBuilder` in favour of PolygonBuilder` and `Ring` has been replaced with `LineStringBuilder`. Also the abstract base classes `BaseLineStringBuilder` and `BasePolygonBuilder` haven been merged with their corresponding implementations. +==== RescoreBuilder + +`RecoreBuilder.Rescorer` was merged with `RescoreBuilder`, which now is an abstract superclass. QueryRescoreBuilder currently is its only implementation. + [[breaking_30_cache_concurrency]] === Cache concurrency level settings removed