diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 14c2bc8ca5a..987087764f7 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -238,7 +238,6 @@ - @@ -415,7 +414,6 @@ - @@ -488,15 +486,12 @@ - - - @@ -555,29 +550,19 @@ - - - - - - - - - - @@ -586,9 +571,7 @@ - - @@ -952,7 +935,6 @@ - @@ -1007,7 +989,6 @@ - @@ -1084,7 +1065,6 @@ - diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java index 1a3f57052de..8fde972e8e4 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableRegistry.java @@ -25,10 +25,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.elasticsearch.plugins.Plugin; /** * A registry for {@link org.elasticsearch.common.io.stream.Writeable.Reader} readers of {@link NamedWriteable}. @@ -47,7 +43,7 @@ public class NamedWriteableRegistry { /** A name for the writeable which is unique to the {@link #categoryClass}. */ public final String name; - /** A reader captability of reading*/ + /** A reader capability of reading*/ public final Writeable.Reader reader; /** Creates a new entry which can be stored by the registry. */ diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index c1932fd4215..0584d37960e 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -871,6 +871,16 @@ public abstract class StreamOutput extends OutputStream { } } + /** + * Writes a list of strings + */ + public void writeStringList(List list) throws IOException { + writeVInt(list.size()); + for (String string: list) { + this.writeString(string); + } + } + /** * Writes a list of {@link NamedWriteable} objects. */ diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index 27b0138e1e5..4516dfde698 100644 --- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -36,7 +36,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; @@ -572,12 +571,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl innerHitsContext.storedFieldsContext(storedFieldsContext); } if (docValueFields != null) { - DocValueFieldsContext docValueFieldsContext = innerHitsContext - .getFetchSubPhaseContext(DocValueFieldsFetchSubPhase.CONTEXT_FACTORY); - for (String field : docValueFields) { - docValueFieldsContext.add(new DocValueFieldsContext.DocValueField(field)); - } - docValueFieldsContext.setHitExecutionNeeded(true); + innerHitsContext.docValueFieldsContext(new DocValueFieldsContext(docValueFields)); } if (scriptFields != null) { for (ScriptField field : scriptFields) { diff --git a/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java b/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java index 97ee715ef6a..364454de0c5 100644 --- a/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java @@ -30,6 +30,8 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParser; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionParser; +import org.elasticsearch.search.SearchExtBuilder; +import org.elasticsearch.search.SearchExtParser; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregator; @@ -82,6 +84,12 @@ public interface SearchPlugin { default List getFetchSubPhases(FetchPhaseConstructionContext context) { return emptyList(); } + /** + * The new {@link SearchExtParser}s defined by this plugin. + */ + default List> getSearchExts() { + return emptyList(); + } /** * Get the {@link Highlighter}s defined by this plugin. */ @@ -160,7 +168,7 @@ public interface SearchPlugin { /** * Specification for an {@link Aggregation}. */ - public static class AggregationSpec extends SearchExtensionSpec { + class AggregationSpec extends SearchExtensionSpec { private final Map> resultReaders = new TreeMap<>(); /** @@ -217,7 +225,7 @@ public interface SearchPlugin { /** * Specification for a {@link PipelineAggregator}. */ - public static class PipelineAggregationSpec extends SearchExtensionSpec { + class PipelineAggregationSpec extends SearchExtensionSpec { private final Map> resultReaders = new TreeMap<>(); private final Writeable.Reader aggregatorReader; @@ -290,6 +298,19 @@ public interface SearchPlugin { } } + /** + * Specification for a {@link SearchExtBuilder} which represents an additional section that can be + * parsed in a search request (within the ext element). + */ + class SearchExtSpec extends SearchExtensionSpec> { + public SearchExtSpec(ParseField name, Writeable.Reader reader, SearchExtParser parser) { + super(name, reader, parser); + } + + public SearchExtSpec(String name, Writeable.Reader reader, SearchExtParser parser) { + super(name, reader, parser); + } + } /** * Specification of search time behavior extension like a custom {@link MovAvgModel} or {@link ScoreFunction}. diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index ae320bccac2..a54e40be731 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -19,10 +19,6 @@ package org.elasticsearch.rest.action.search; -import java.io.IOException; -import java.util.Map; -import java.util.function.BiConsumer; - import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.SearchRequest; @@ -40,12 +36,16 @@ import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.builder.SearchSourceBuilder; +import java.io.IOException; +import java.util.Map; +import java.util.function.BiConsumer; + import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue; @@ -97,7 +97,7 @@ public class RestMultiSearchAction extends BaseRestHandler { final QueryParseContext queryParseContext = new QueryParseContext(searchRequestParsers.queryParsers, requestParser, parseFieldMatcher); searchRequest.source(SearchSourceBuilder.fromXContent(queryParseContext, - searchRequestParsers.aggParsers, searchRequestParsers.suggesters)); + searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers)); multiRequest.add(searchRequest); } catch (IOException e) { throw new ElasticsearchParseException("Exception when parsing search request", e); diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 215165e6bbf..8acfc72dfe1 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -102,7 +102,8 @@ public class RestSearchAction extends BaseRestHandler { if (restContent != null) { try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { QueryParseContext context = new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher); - searchRequest.source().parseXContent(context, searchRequestParsers.aggParsers, searchRequestParsers.suggesters); + searchRequest.source().parseXContent(context, searchRequestParsers.aggParsers, searchRequestParsers.suggesters, + searchRequestParsers.searchExtParsers); } } diff --git a/core/src/main/java/org/elasticsearch/search/SearchExtBuilder.java b/core/src/main/java/org/elasticsearch/search/SearchExtBuilder.java new file mode 100644 index 00000000000..8d75216fe12 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/SearchExtBuilder.java @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search; + +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.plugins.SearchPlugin.SearchExtSpec; + +/** + * Intermediate serializable representation of a search ext section. To be subclassed by plugins that support + * a custom section as part of a search request, which will be provided within the ext element. + * Any state needs to be serialized as part of the {@link Writeable#writeTo(StreamOutput)} method and + * read from the incoming stream, usually done adding a constructor that takes {@link StreamInput} as + * an argument. + * + * Registration happens through {@link SearchPlugin#getSearchExts()}, which also needs a {@link SearchExtParser} that's able to parse + * the incoming request from the REST layer into the proper {@link SearchExtBuilder} subclass. + * + * {@link #getWriteableName()} must return the same name as the one used for the registration + * of the {@link SearchExtSpec}. + * + * @see SearchExtParser + * @see SearchExtSpec + */ +public abstract class SearchExtBuilder implements NamedWriteable, ToXContent { + + public abstract int hashCode(); + + public abstract boolean equals(Object obj); +} diff --git a/core/src/main/java/org/elasticsearch/search/SearchExtParser.java b/core/src/main/java/org/elasticsearch/search/SearchExtParser.java new file mode 100644 index 00000000000..a2fe4cfe0cb --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/SearchExtParser.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search; + +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +/** + * Defines a parser that is able to parse {@link org.elasticsearch.search.SearchExtBuilder}s + * from {@link org.elasticsearch.common.xcontent.XContent}. + * + * Registration happens through {@link org.elasticsearch.plugins.SearchPlugin#getSearchExts()}, which also needs a {@link SearchExtBuilder} + * implementation which is the object that this parser returns when reading an incoming request form the REST layer. + * + * @see SearchExtBuilder + * @see org.elasticsearch.plugins.SearchPlugin.SearchExtSpec + */ +@FunctionalInterface +public interface SearchExtParser { + + /** + * Parses the supported element placed within the ext section of a search request + */ + T fromXContent(XContentParser parser) throws IOException; +} diff --git a/core/src/main/java/org/elasticsearch/search/SearchParseElement.java b/core/src/main/java/org/elasticsearch/search/SearchExtRegistry.java similarity index 76% rename from core/src/main/java/org/elasticsearch/search/SearchParseElement.java rename to core/src/main/java/org/elasticsearch/search/SearchExtRegistry.java index 9bf680deb55..dd04145ba7d 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/SearchExtRegistry.java @@ -19,13 +19,14 @@ package org.elasticsearch.search; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.common.xcontent.ParseFieldRegistry; /** - * + * Extensions to ParseFieldRegistry to make Guice happy. */ -public interface SearchParseElement { +public class SearchExtRegistry extends ParseFieldRegistry { - void parse(XContentParser parser, SearchContext context) throws Exception; + public SearchExtRegistry() { + super("ext"); + } } diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java index ab36f03639c..0a1d9824ea9 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java @@ -93,6 +93,7 @@ import org.elasticsearch.plugins.SearchPlugin.FetchPhaseConstructionContext; import org.elasticsearch.plugins.SearchPlugin.PipelineAggregationSpec; import org.elasticsearch.plugins.SearchPlugin.QuerySpec; import org.elasticsearch.plugins.SearchPlugin.ScoreFunctionSpec; +import org.elasticsearch.plugins.SearchPlugin.SearchExtSpec; import org.elasticsearch.plugins.SearchPlugin.SearchExtensionSpec; import org.elasticsearch.search.action.SearchTransportService; import org.elasticsearch.search.aggregations.AggregationBuilder; @@ -306,6 +307,7 @@ public class SearchModule extends AbstractModule { "moving_avg_model"); private final List fetchSubPhases = new ArrayList<>(); + private final SearchExtRegistry searchExtParserRegistry = new SearchExtRegistry(); private final Settings settings; private final List namedWriteables = new ArrayList<>(); @@ -326,8 +328,9 @@ public class SearchModule extends AbstractModule { registerAggregations(plugins); registerPipelineAggregations(plugins); registerFetchSubPhases(plugins); + registerSearchExts(plugins); registerShapes(); - searchRequestParsers = new SearchRequestParsers(queryParserRegistry, aggregatorParsers, getSuggesters()); + searchRequestParsers = new SearchRequestParsers(queryParserRegistry, aggregatorParsers, getSuggesters(), searchExtParserRegistry); } public List getNamedWriteables() { @@ -380,6 +383,7 @@ public class SearchModule extends AbstractModule { if (false == transportClient) { bind(IndicesQueriesRegistry.class).toInstance(queryParserRegistry); bind(SearchRequestParsers.class).toInstance(searchRequestParsers); + bind(SearchExtRegistry.class).toInstance(searchExtParserRegistry); configureSearch(); } } @@ -725,6 +729,15 @@ public class SearchModule extends AbstractModule { registerFromPlugin(plugins, p -> p.getFetchSubPhases(context), this::registerFetchSubPhase); } + private void registerSearchExts(List plugins) { + registerFromPlugin(plugins, SearchPlugin::getSearchExts, this::registerSearchExt); + } + + private void registerSearchExt(SearchExtSpec spec) { + searchExtParserRegistry.register(spec.getParser(), spec.getName()); + namedWriteables.add(new Entry(SearchExtBuilder.class, spec.getName().getPreferredName(), spec.getReader())); + } + private void registerFetchSubPhase(FetchSubPhase subPhase) { Class subPhaseClass = subPhase.getClass(); if (fetchSubPhases.stream().anyMatch(p -> p.getClass().equals(subPhaseClass))) { diff --git a/core/src/main/java/org/elasticsearch/search/SearchPhase.java b/core/src/main/java/org/elasticsearch/search/SearchPhase.java index 48c041f12f5..33260706b3c 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/SearchPhase.java @@ -21,22 +21,18 @@ package org.elasticsearch.search; import org.elasticsearch.search.internal.SearchContext; -import java.util.Collections; -import java.util.Map; - /** - * + * Represents a phase of a search request e.g. query, fetch etc. */ public interface SearchPhase { - default Map parseElements() { - return Collections.emptyMap(); - } - /** * Performs pre processing of the search context before the execute. */ void preProcess(SearchContext context); + /** + * Executes the search phase + */ void execute(SearchContext context); } diff --git a/core/src/main/java/org/elasticsearch/search/SearchRequestParsers.java b/core/src/main/java/org/elasticsearch/search/SearchRequestParsers.java index 83eebd125d8..ba3afc15771 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchRequestParsers.java +++ b/core/src/main/java/org/elasticsearch/search/SearchRequestParsers.java @@ -37,7 +37,8 @@ public class SearchRequestParsers { /** * Query parsers that may be used in search requests. * @see org.elasticsearch.index.query.QueryParseContext - * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers, Suggesters) + * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers, + * Suggesters, SearchExtRegistry) */ public final IndicesQueriesRegistry queryParsers; @@ -45,20 +46,29 @@ public class SearchRequestParsers { // and pipeline agg parsers should be here /** * Agg and pipeline agg parsers that may be used in search requests. - * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers, Suggesters) + * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers, + * Suggesters, SearchExtRegistry) */ public final AggregatorParsers aggParsers; // TODO: Suggesters should be removed and the underlying map moved here /** * Suggesters that may be used in search requests. - * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers, Suggesters) + * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers, + * Suggesters, SearchExtRegistry) */ public final Suggesters suggesters; - public SearchRequestParsers(IndicesQueriesRegistry queryParsers, AggregatorParsers aggParsers, Suggesters suggesters) { + /** + * Pluggable section that can be parsed out of a search section, within the ext element + */ + public final SearchExtRegistry searchExtParsers; + + public SearchRequestParsers(IndicesQueriesRegistry queryParsers, AggregatorParsers aggParsers, Suggesters suggesters, + SearchExtRegistry searchExtParsers) { this.queryParsers = queryParsers; this.aggParsers = aggParsers; this.suggesters = suggesters; + this.searchExtParsers = searchExtParsers; } } diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index ba39c55a7a2..2fb87565aa3 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -39,9 +39,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.ConcurrentMapLong; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; @@ -67,8 +64,6 @@ import org.elasticsearch.search.fetch.QueryFetchSearchResult; import org.elasticsearch.search.fetch.ScrollQueryFetchSearchResult; import org.elasticsearch.search.fetch.ShardFetchRequest; import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext.DocValueField; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.internal.DefaultSearchContext; @@ -102,7 +97,6 @@ import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; -import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes; @@ -145,8 +139,6 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv private final ConcurrentMapLong activeContexts = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency(); - private final Map elementParsers; - private final ParseFieldMatcher parseFieldMatcher; @Inject @@ -165,12 +157,6 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv TimeValue keepAliveInterval = KEEPALIVE_INTERVAL_SETTING.get(settings); this.defaultKeepAlive = DEFAULT_KEEPALIVE_SETTING.get(settings).millis(); - Map elementParsers = new HashMap<>(); - elementParsers.putAll(dfsPhase.parseElements()); - elementParsers.putAll(queryPhase.parseElements()); - elementParsers.putAll(fetchPhase.parseElements()); - this.elementParsers = unmodifiableMap(elementParsers); - this.keepAliveReaper = threadPool.scheduleWithFixedDelay(new Reaper(), keepAliveInterval, Names.SAME); defaultSearchTimeout = DEFAULT_SEARCH_TIMEOUT_SETTING.get(settings); @@ -465,7 +451,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv throw ExceptionsHelper.convertToRuntime(e); } operationListener.onFetchPhase(context, System.nanoTime() - time2); - return new ScrollQueryFetchSearchResult(new QueryFetchSearchResult(context.queryResult(), context.fetchResult()), context.shardTarget()); + return new ScrollQueryFetchSearchResult(new QueryFetchSearchResult(context.queryResult(), context.fetchResult()), + context.shardTarget()); } catch (Exception e) { logger.trace("Fetch phase failed", e); processFailure(context, e); @@ -736,11 +723,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv context.fetchSourceContext(source.fetchSource()); } if (source.docValueFields() != null) { - DocValueFieldsContext docValuesFieldsContext = context.getFetchSubPhaseContext(DocValueFieldsFetchSubPhase.CONTEXT_FACTORY); - for (String field : source.docValueFields()) { - docValuesFieldsContext.add(new DocValueField(field)); - } - docValuesFieldsContext.setHitExecutionNeeded(true); + context.docValueFieldsContext(new DocValueFieldsContext(source.docValueFields())); } if (source.highlighter() != null) { HighlightBuilder highlightBuilder = source.highlighter(); @@ -758,47 +741,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv } } if (source.ext() != null) { - XContentParser extParser = null; - try { - extParser = XContentFactory.xContent(source.ext()).createParser(source.ext()); - if (extParser.nextToken() != XContentParser.Token.START_OBJECT) { - throw new SearchParseException(context, "expected start object, found [" + extParser.currentToken() + "] instead", - extParser.getTokenLocation()); - } - XContentParser.Token token; - String currentFieldName = null; - while ((token = extParser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = extParser.currentName(); - } else { - SearchParseElement parseElement = this.elementParsers.get(currentFieldName); - if (parseElement == null) { - if (currentFieldName != null && currentFieldName.equals("suggest")) { - throw new SearchParseException(context, - "suggest is not supported in [ext], please use SearchSourceBuilder#suggest(SuggestBuilder) instead", - extParser.getTokenLocation()); - } - throw new SearchParseException(context, "Unknown element [" + currentFieldName + "] in [ext]", - extParser.getTokenLocation()); - } else { - parseElement.parse(extParser, context); - } - } - } - } catch (Exception e) { - String sSource = "_na_"; - try { - sSource = source.toString(); - } catch (Exception inner) { - e.addSuppressed(inner); - // ignore - } - XContentLocation location = extParser != null ? extParser.getTokenLocation() : null; - throw new SearchParseException(context, "failed to parse ext source [" + sSource + "]", location, e); - } finally { - if (extParser != null) { - extParser.close(); - } + for (SearchExtBuilder searchExtBuilder : source.ext()) { + context.addSearchExt(searchExtBuilder); } } if (source.version() != null) { @@ -914,7 +858,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv continue; } if ((time - lastAccessTime > context.keepAlive())) { - logger.debug("freeing search context [{}], time [{}], lastAccessTime [{}], keepAlive [{}]", context.id(), time, lastAccessTime, context.keepAlive()); + logger.debug("freeing search context [{}], time [{}], lastAccessTime [{}], keepAlive [{}]", context.id(), time, + lastAccessTime, context.keepAlive()); freeContext(context.id()); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java index 5dc29374310..8acd4f13752 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java @@ -40,7 +40,7 @@ import java.util.Collections; import java.util.List; /** - * + * Aggregation phase of a search request, used to collect aggregations */ public class AggregationPhase implements SearchPhase { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java index b3389322d9c..7c6a743a20b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.metrics.tophits; -import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.aggregations.Aggregator; @@ -29,9 +28,8 @@ import org.elasticsearch.search.aggregations.InternalAggregation.Type; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; +import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext.DocValueField; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SubSearchContext; @@ -98,12 +96,7 @@ public class TopHitsAggregatorFactory extends AggregatorFactory> rescoreBuilders; + private List rescoreBuilders; private ObjectFloatHashMap indexBoost = null; private List stats; - private BytesReference ext = null; + private List extBuilders = Collections.emptyList(); private boolean profile = false; @@ -203,18 +203,10 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ postQueryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); if (in.readBoolean()) { - int size = in.readVInt(); - rescoreBuilders = new ArrayList<>(); - for (int i = 0; i < size; i++) { - rescoreBuilders.add(in.readNamedWriteable(RescoreBuilder.class)); - } + rescoreBuilders = in.readNamedWriteableList(RescoreBuilder.class); } if (in.readBoolean()) { - int size = in.readVInt(); - scriptFields = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - scriptFields.add(new ScriptField(in)); - } + scriptFields = in.readList(ScriptField::new); } size = in.readVInt(); if (in.readBoolean()) { @@ -225,18 +217,14 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } } if (in.readBoolean()) { - int size = in.readVInt(); - stats = new ArrayList<>(); - for (int i = 0; i < size; i++) { - stats.add(in.readString()); - } + stats = in.readList(StreamInput::readString); } suggestBuilder = in.readOptionalWriteable(SuggestBuilder::new); terminateAfter = in.readVInt(); timeout = in.readOptionalWriteable(TimeValue::new); trackScores = in.readBoolean(); version = in.readOptionalBoolean(); - ext = in.readOptionalBytesReference(); + extBuilders = in.readNamedWriteableList(SearchExtBuilder.class); profile = in.readBoolean(); searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new); sliceBuilder = in.readOptionalWriteable(SliceBuilder::new); @@ -262,18 +250,12 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ boolean hasRescoreBuilders = rescoreBuilders != null; out.writeBoolean(hasRescoreBuilders); if (hasRescoreBuilders) { - out.writeVInt(rescoreBuilders.size()); - for (RescoreBuilder rescoreBuilder : rescoreBuilders) { - out.writeNamedWriteable(rescoreBuilder); - } + out.writeNamedWriteableList(rescoreBuilders); } boolean hasScriptFields = scriptFields != null; out.writeBoolean(hasScriptFields); if (hasScriptFields) { - out.writeVInt(scriptFields.size()); - for (ScriptField scriptField : scriptFields) { - scriptField.writeTo(out); - } + out.writeList(scriptFields); } out.writeVInt(size); boolean hasSorts = sorts != null; @@ -287,17 +269,14 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ boolean hasStats = stats != null; out.writeBoolean(hasStats); if (hasStats) { - out.writeVInt(stats.size()); - for (String stat : stats) { - out.writeString(stat); - } + out.writeStringList(stats); } out.writeOptionalWriteable(suggestBuilder); out.writeVInt(terminateAfter); out.writeOptionalWriteable(timeout); out.writeBoolean(trackScores); out.writeOptionalBoolean(version); - out.writeOptionalBytesReference(ext); + out.writeNamedWriteableList(extBuilders); out.writeBoolean(profile); out.writeOptionalWriteable(searchAfterBuilder); out.writeOptionalWriteable(sliceBuilder); @@ -649,7 +628,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ /** * Gets the bytes representing the rescore builders for this request. */ - public List> rescores() { + public List rescores() { return rescoreBuilders; } @@ -875,13 +854,13 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ return stats; } - public SearchSourceBuilder ext(XContentBuilder ext) { - this.ext = ext.bytes(); + public SearchSourceBuilder ext(List searchExtBuilders) { + this.extBuilders = Objects.requireNonNull(searchExtBuilders, "searchExtBuilders must not be null"); return this; } - public BytesReference ext() { - return ext; + public List ext() { + return extBuilders; } /** @@ -919,7 +898,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ SearchSourceBuilder rewrittenBuilder = new SearchSourceBuilder(); rewrittenBuilder.aggregations = aggregations; rewrittenBuilder.explain = explain; - rewrittenBuilder.ext = ext; + rewrittenBuilder.extBuilders = extBuilders; rewrittenBuilder.fetchSourceContext = fetchSourceContext; rewrittenBuilder.docValueFields = docValueFields; rewrittenBuilder.storedFieldsContext = storedFieldsContext; @@ -948,17 +927,18 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ /** * Parse some xContent into this SearchSourceBuilder, overwriting any values specified in the xContent. Use this if you need to set up * different defaults than a regular SearchSourceBuilder would have and use - * {@link #fromXContent(QueryParseContext, AggregatorParsers, Suggesters)} if you have normal defaults. + * {@link #fromXContent(QueryParseContext, AggregatorParsers, Suggesters, SearchExtRegistry)} if you have normal defaults. */ - public void parseXContent(QueryParseContext context, AggregatorParsers aggParsers, Suggesters suggesters) + public void parseXContent(QueryParseContext context, AggregatorParsers aggParsers, + Suggesters suggesters, SearchExtRegistry searchExtRegistry) throws IOException { XContentParser parser = context.parser(); XContentParser.Token token = parser.currentToken(); String currentFieldName = null; if (token != XContentParser.Token.START_OBJECT && (token = parser.nextToken()) != XContentParser.Token.START_OBJECT) { - throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT + "] but found [" + token + "]", - parser.getTokenLocation()); + throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT + + "] but found [" + token + "]", parser.getTokenLocation()); } while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -1017,8 +997,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } else if (token.isValue()) { indexBoost.put(currentFieldName, parser.floatValue()); } else { - throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", - parser.getTokenLocation()); + throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + + " in [" + currentFieldName + "].", parser.getTokenLocation()); } } } else if (context.getParseFieldMatcher().match(currentFieldName, AGGREGATIONS_FIELD) @@ -1034,8 +1014,23 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ rescoreBuilders = new ArrayList<>(); rescoreBuilders.add(RescoreBuilder.parseFromXContent(context)); } else if (context.getParseFieldMatcher().match(currentFieldName, EXT_FIELD)) { - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); - ext = xContentBuilder.bytes(); + extBuilders = new ArrayList<>(); + String extSectionName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + extSectionName = parser.currentName(); + } else { + SearchExtParser searchExtParser = searchExtRegistry.lookup(extSectionName, + context.getParseFieldMatcher(), parser.getTokenLocation()); + SearchExtBuilder searchExtBuilder = searchExtParser.fromXContent(parser); + if (searchExtBuilder.getWriteableName().equals(extSectionName) == false) { + throw new IllegalStateException("The parsed [" + searchExtBuilder.getClass().getName() + "] object has a " + + "different writeable name compared to the name of the section that it was parsed from: found [" + + searchExtBuilder.getWriteableName() + "] expected [" + extSectionName + "]"); + } + extBuilders.add(searchExtBuilder); + } + } } else if (context.getParseFieldMatcher().match(currentFieldName, SLICE)) { sliceBuilder = SliceBuilder.fromXContent(context); } else { @@ -1051,8 +1046,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ if (token == XContentParser.Token.VALUE_STRING) { docValueFields.add(parser.text()); } else { - throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in [" - + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation()); + throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + + "] in [" + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation()); } } } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_FIELD)) { @@ -1068,8 +1063,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ if (token == XContentParser.Token.VALUE_STRING) { stats.add(parser.text()); } else { - throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in [" - + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation()); + throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + + "] in [" + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation()); } } } else if (context.getParseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) { @@ -1221,12 +1216,12 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ builder.field(STATS_FIELD.getPreferredName(), stats); } - if (ext != null) { - builder.field(EXT_FIELD.getPreferredName()); - try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(ext)) { - parser.nextToken(); - builder.copyCurrentStructure(parser); + if (extBuilders != null) { + builder.startObject(EXT_FIELD.getPreferredName()); + for (SearchExtBuilder extBuilder : extBuilders) { + extBuilder.toXContent(builder, params); } + builder.endObject(); } } @@ -1344,9 +1339,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ @Override public int hashCode() { - return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldsContext, from, - highlightBuilder, indexBoost, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, - size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, profile); + return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder, + indexBoost, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size, sorts, searchAfterBuilder, + sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, profile, extBuilders); } @Override @@ -1381,7 +1376,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ && Objects.equals(timeout, other.timeout) && Objects.equals(trackScores, other.trackScores) && Objects.equals(version, other.version) - && Objects.equals(profile, other.profile); + && Objects.equals(profile, other.profile) + && Objects.equals(extBuilders, other.extBuilders); } - } diff --git a/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java b/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java index de06655f414..1359be24a15 100644 --- a/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java +++ b/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java @@ -37,7 +37,8 @@ import java.util.Collection; import java.util.Iterator; /** - * + * Dfs phase of a search request, used to make scoring 100% accurate by collecting additional info from each shard before the query phase. + * The additional information is used to better compare the scores coming from all the shards, which depend on local factors (e.g. idf) */ public class DfsPhase implements SearchPhase { diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 997cb7caa4b..41ea0e294da 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -43,7 +43,6 @@ import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHitField; -import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.SearchPhase; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.InnerHitsFetchSubPhase; @@ -62,11 +61,11 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.common.xcontent.XContentFactory.contentBuilder; /** - * + * Fetch phase of a search request, used to fetch the actual top matching documents to be returned to the client, identified + * after reducing all of the matches returned by the query phase */ public class FetchPhase implements SearchPhase { @@ -77,15 +76,6 @@ public class FetchPhase implements SearchPhase { this.fetchSubPhases[fetchSubPhases.size()] = new InnerHitsFetchSubPhase(this); } - @Override - public Map parseElements() { - Map parseElements = new HashMap<>(); - for (FetchSubPhase fetchSubPhase : fetchSubPhases) { - parseElements.putAll(fetchSubPhase.parseElements()); - } - return unmodifiableMap(parseElements); - } - @Override public void preProcess(SearchContext context) { } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index 2180cc34128..1783652d120 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -22,11 +22,9 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexSearcher; -import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.SearchContext; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -69,10 +67,6 @@ public interface FetchSubPhase { return searcher.getIndexReader(); } - public IndexSearcher topLevelSearcher() { - return searcher; - } - public Map cache() { if (cache == null) { cache = new HashMap<>(); @@ -82,10 +76,6 @@ public interface FetchSubPhase { } - default Map parseElements() { - return Collections.emptyMap(); - } - /** * Executes the hit level phase, with a reader and doc id (note, its a low level reader, and the matching doc). */ @@ -93,23 +83,4 @@ public interface FetchSubPhase { default void hitsExecute(SearchContext context, InternalSearchHit[] hits) {} - - /** - * This interface is in the fetch phase plugin mechanism. - * Whenever a new search is executed we create a new {@link SearchContext} that holds individual contexts for each {@link org.elasticsearch.search.fetch.FetchSubPhase}. - * Fetch phases that use the plugin mechanism must provide a ContextFactory to the SearchContext that creates the fetch phase context and also associates them with a name. - * See {@link SearchContext#getFetchSubPhaseContext(FetchSubPhase.ContextFactory)} - */ - interface ContextFactory { - - /** - * The name of the context. - */ - String getName(); - - /** - * Creates a new instance of a FetchSubPhaseContext that holds all information a FetchSubPhase needs to execute on hits. - */ - SubPhaseContext newContextInstance(); - } } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseContext.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseContext.java deleted file mode 100644 index 856c0ad902f..00000000000 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseContext.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.fetch; - -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; - -/** - * All configuration and context needed by the FetchSubPhase to execute on hits. - * The only required information in this base class is whether or not the sub phase needs to be run at all. - * It can be extended by FetchSubPhases to hold information the phase needs to execute on hits. - * See {@link org.elasticsearch.search.fetch.FetchSubPhase.ContextFactory} and also {@link DocValueFieldsContext} for an example. - */ -public class FetchSubPhaseContext { - - // This is to store if the FetchSubPhase should be executed at all. - private boolean hitExecutionNeeded = false; - - /** - * Set if this phase should be executed at all. - */ - public void setHitExecutionNeeded(boolean hitExecutionNeeded) { - this.hitExecutionNeeded = hitExecutionNeeded; - } - - /** - * Returns if this phase be executed at all. - */ - public boolean hitExecutionNeeded() { - return hitExecutionNeeded; - } - -} diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java index 54185734f97..325d28e4592 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java @@ -18,38 +18,23 @@ */ package org.elasticsearch.search.fetch.subphase; -import org.elasticsearch.search.fetch.FetchSubPhaseContext; - -import java.util.ArrayList; import java.util.List; /** * All the required context to pull a field from the doc values. */ -public class DocValueFieldsContext extends FetchSubPhaseContext { +public class DocValueFieldsContext { - public static class DocValueField { - private final String name; + private final List fields; - public DocValueField(String name) { - this.name = name; - } - - public String name() { - return name; - } + public DocValueFieldsContext(List fields) { + this.fields = fields; } - private List fields = new ArrayList<>(); - - public DocValueFieldsContext() { - } - - public void add(DocValueField field) { - this.fields.add(field); - } - - public List fields() { + /** + * Returns the required docvalue fields + */ + public List fields() { return this.fields; } } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java index 803cbb4348f..befce94a9e4 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java @@ -36,35 +36,21 @@ import java.util.HashMap; */ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase { - public static final String NAME = "docvalue_fields"; - public static final ContextFactory CONTEXT_FACTORY = new ContextFactory() { - - @Override - public String getName() { - return NAME; - } - - @Override - public DocValueFieldsContext newContextInstance() { - return new DocValueFieldsContext(); - } - }; - @Override public void hitExecute(SearchContext context, HitContext hitContext) { - if (context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded() == false) { + if (context.docValueFieldsContext() == null) { return; } - for (DocValueFieldsContext.DocValueField field : context.getFetchSubPhaseContext(CONTEXT_FACTORY).fields()) { + for (String field : context.docValueFieldsContext().fields()) { if (hitContext.hit().fieldsOrNull() == null) { hitContext.hit().fields(new HashMap<>(2)); } - SearchHitField hitField = hitContext.hit().fields().get(field.name()); + SearchHitField hitField = hitContext.hit().fields().get(field); if (hitField == null) { - hitField = new InternalSearchHitField(field.name(), new ArrayList<>(2)); - hitContext.hit().fields().put(field.name(), hitField); + hitField = new InternalSearchHitField(field, new ArrayList<>(2)); + hitContext.hit().fields().put(field, hitField); } - MappedFieldType fieldType = context.mapperService().fullName(field.name()); + MappedFieldType fieldType = context.mapperService().fullName(field); if (fieldType != null) { AtomicFieldData data = context.fieldData().getForField(fieldType).load(hitContext.readerContext()); ScriptDocValues values = data.getScriptValues(); diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java index 780352508bf..b1618c3a205 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java @@ -50,17 +50,17 @@ import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.SearchExtBuilder; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.aggregations.SearchContextAggregations; import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.FetchSearchResult; -import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.fetch.FetchSubPhaseContext; +import org.elasticsearch.search.fetch.StoredFieldsContext; +import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight; @@ -80,9 +80,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -/** - * - */ public class DefaultSearchContext extends SearchContext { private final long id; @@ -110,6 +107,7 @@ public class DefaultSearchContext extends SearchContext { private StoredFieldsContext storedFields; private ScriptFieldsContext scriptFields; private FetchSourceContext fetchSourceContext; + private DocValueFieldsContext docValueFieldsContext; private int from = -1; private int size = -1; private SortAndFormats sort; @@ -148,7 +146,7 @@ public class DefaultSearchContext extends SearchContext { private volatile long lastAccessTime = -1; private Profilers profilers; - private final Map subPhaseContexts = new HashMap<>(); + private final Map searchExtBuilders = new HashMap<>(); private final Map, Collector> queryCollectors = new HashMap<>(); private final QueryShardContext queryShardContext; private FetchPhase fetchPhase; @@ -388,14 +386,16 @@ public class DefaultSearchContext extends SearchContext { } @Override - public SubPhaseContext getFetchSubPhaseContext(FetchSubPhase.ContextFactory contextFactory) { - String subPhaseName = contextFactory.getName(); - if (subPhaseContexts.get(subPhaseName) == null) { - subPhaseContexts.put(subPhaseName, contextFactory.newContextInstance()); - } - return (SubPhaseContext) subPhaseContexts.get(subPhaseName); + public void addSearchExt(SearchExtBuilder searchExtBuilder) { + //it's ok to use the writeable name here given that we enforce it to be the same as the name of the element that gets + //parsed by the corresponding parser. There is one single name and one single way to retrieve the parsed object from the context. + searchExtBuilders.put(searchExtBuilder.getWriteableName(), searchExtBuilder); } + @Override + public SearchExtBuilder getSearchExt(String name) { + return searchExtBuilders.get(name); + } @Override public SearchContextHighlight highlight() { @@ -470,6 +470,17 @@ public class DefaultSearchContext extends SearchContext { return this; } + @Override + public DocValueFieldsContext docValueFieldsContext() { + return docValueFieldsContext; + } + + @Override + public SearchContext docValueFieldsContext(DocValueFieldsContext docValueFieldsContext) { + this.docValueFieldsContext = docValueFieldsContext; + return this; + } + @Override public ContextIndexSearcher searcher() { return this.searcher; diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 4e52ab0d534..18af4873db3 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -35,17 +35,16 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.SearchExtBuilder; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.aggregations.SearchContextAggregations; import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.FetchSearchResult; -import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.fetch.FetchSubPhaseContext; +import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; @@ -512,8 +511,13 @@ public abstract class FilteredSearchContext extends SearchContext { } @Override - public SubPhaseContext getFetchSubPhaseContext(FetchSubPhase.ContextFactory contextFactory) { - return in.getFetchSubPhaseContext(contextFactory); + public void addSearchExt(SearchExtBuilder searchExtBuilder) { + in.addSearchExt(searchExtBuilder); + } + + @Override + public SearchExtBuilder getSearchExt(String name) { + return in.getSearchExt(name); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 65fe7ddad15..e96c9856a33 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -22,9 +22,7 @@ package org.elasticsearch.search.internal; import org.apache.lucene.search.Collector; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.Query; -import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.Counter; -import org.apache.lucene.util.RefCount; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseFieldMatcher; @@ -43,17 +41,17 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.SearchExtBuilder; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.aggregations.SearchContextAggregations; import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.FetchSearchResult; -import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.fetch.FetchSubPhaseContext; +import org.elasticsearch.search.fetch.StoredFieldsContext; +import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; @@ -187,7 +185,9 @@ public abstract class SearchContext extends AbstractRefCounted implements Releas public abstract SearchContext aggregations(SearchContextAggregations aggregations); - public abstract SubPhaseContext getFetchSubPhaseContext(FetchSubPhase.ContextFactory contextFactory); + public abstract void addSearchExt(SearchExtBuilder searchExtBuilder); + + public abstract SearchExtBuilder getSearchExt(String name); public abstract SearchContextHighlight highlight(); @@ -226,6 +226,10 @@ public abstract class SearchContext extends AbstractRefCounted implements Releas public abstract SearchContext fetchSourceContext(FetchSourceContext fetchSourceContext); + public abstract DocValueFieldsContext docValueFieldsContext(); + + public abstract SearchContext docValueFieldsContext(DocValueFieldsContext docValueFieldsContext); + public abstract ContextIndexSearcher searcher(); public abstract IndexShard indexShard(); diff --git a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java index 077a0941a0b..dc385a0e120 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.search.aggregations.SearchContextAggregations; import org.elasticsearch.search.fetch.FetchSearchResult; +import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight; @@ -36,8 +37,6 @@ import org.elasticsearch.search.suggest.SuggestionSearchContext; import java.util.List; -/** - */ public class SubSearchContext extends FilteredSearchContext { // By default return 3 hits per bucket. A higher default would make the response really large by default, since @@ -60,6 +59,7 @@ public class SubSearchContext extends FilteredSearchContext { private StoredFieldsContext storedFields; private ScriptFieldsContext scriptFields; private FetchSourceContext fetchSourceContext; + private DocValueFieldsContext docValueFieldsContext; private SearchContextHighlight highlight; private boolean explain; @@ -154,6 +154,17 @@ public class SubSearchContext extends FilteredSearchContext { return this; } + @Override + public DocValueFieldsContext docValueFieldsContext() { + return docValueFieldsContext; + } + + @Override + public SearchContext docValueFieldsContext(DocValueFieldsContext docValueFieldsContext) { + this.docValueFieldsContext = docValueFieldsContext; + return this; + } + @Override public void timeout(TimeValue timeout) { throw new UnsupportedOperationException("Not supported"); diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 189fead7813..47fa98856cf 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -68,7 +68,8 @@ import java.util.List; import java.util.concurrent.Callable; /** - * + * Query phase of a search request, used to run the query and get back from each shard information about the matching documents + * (document ids and score or sort criteria) so that matches can be reduced on the coordinating node */ public class QueryPhase implements SearchPhase { diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java index 395db4cdcd8..d3d4c75cd7b 100644 --- a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java +++ b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java @@ -29,6 +29,7 @@ import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; /** + * Rescore phase of a search request, used to run potentially expensive scoring models against the top matching documents. */ public class RescorePhase extends AbstractComponent implements SearchPhase { diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java index c0567e59e8c..874448b924c 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Map; /** + * Suggest phase of a search request, used to collect suggestions */ public class SuggestPhase extends AbstractComponent implements SearchPhase { diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java index 3c9acb4104b..d5b17861e0f 100644 --- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java @@ -175,6 +175,6 @@ public class MultiSearchRequestTests extends ESTestCase { IndicesQueriesRegistry registry = new IndicesQueriesRegistry(); QueryParser parser = MatchAllQueryBuilder::fromXContent; registry.register(parser, MatchAllQueryBuilder.NAME); - return new SearchRequestParsers(registry, null, null); + return new SearchRequestParsers(registry, null, null, null); } } diff --git a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java index 3db3492f415..548a09d37bc 100644 --- a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java +++ b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.search.fetch.FetchSubPhasePluginIT; import org.elasticsearch.test.ESTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -36,6 +37,7 @@ import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import static java.util.Collections.emptyList; @@ -53,7 +55,8 @@ public class SearchRequestTests extends ESTestCase { bindMapperExtension(); } }; - SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()) { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, + Collections.singletonList(new FetchSubPhasePluginIT.FetchTermVectorsPlugin())) { @Override protected void configureSearch() { // Skip me diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 0742fca357d..7960e9bf4e8 100644 --- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -59,6 +59,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.fetch.FetchSubPhasePluginIT; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests; import org.elasticsearch.search.rescore.QueryRescoreBuilderTests; @@ -85,7 +86,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static java.util.Collections.emptyList; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.elasticsearch.test.ClusterServiceUtils.setState; import static org.hamcrest.CoreMatchers.containsString; @@ -132,7 +132,8 @@ public class SearchSourceBuilderTests extends ESTestCase { bindMapperExtension(); } }; - SearchModule searchModule = new SearchModule(settings, false, emptyList()) { + SearchModule searchModule = new SearchModule(settings, false, + Collections.singletonList(new FetchSubPhasePluginIT.FetchTermVectorsPlugin())) { @Override protected void configureSearch() { // Skip me @@ -389,11 +390,7 @@ public class SearchSourceBuilderTests extends ESTestCase { builder.aggregation(AggregationBuilders.avg(randomAsciiOfLengthBetween(5, 20))); } if (randomBoolean()) { - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(); - xContentBuilder.startObject(); - xContentBuilder.field("term_vectors_fetch", randomAsciiOfLengthBetween(5, 20)); - xContentBuilder.endObject(); - builder.ext(xContentBuilder); + builder.ext(Collections.singletonList(new FetchSubPhasePluginIT.TermVectorsFetchBuilder("test"))); } if (randomBoolean()) { String field = randomBoolean() ? null : randomAsciiOfLengthBetween(5, 20); @@ -431,15 +428,14 @@ public class SearchSourceBuilderTests extends ESTestCase { // test the embedded case } SearchSourceBuilder newBuilder = SearchSourceBuilder.fromXContent(parseContext, searchRequestParsers.aggParsers, - searchRequestParsers.suggesters); + searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); assertNull(parser.nextToken()); assertEquals(testBuilder, newBuilder); assertEquals(testBuilder.hashCode(), newBuilder.hashCode()); } private static QueryParseContext createParseContext(XContentParser parser) { - QueryParseContext context = new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher); - return context; + return new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher); } public void testSerialization() throws IOException { @@ -497,7 +493,7 @@ public class SearchSourceBuilderTests extends ESTestCase { String restContent = " { \"_source\": { \"includes\": \"include\", \"excludes\": \"*.field2\"}}"; try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser), - searchRequestParsers.aggParsers, searchRequestParsers.suggesters); + searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); assertArrayEquals(new String[]{"*.field2"}, searchSourceBuilder.fetchSource().excludes()); assertArrayEquals(new String[]{"include"}, searchSourceBuilder.fetchSource().includes()); } @@ -506,7 +502,7 @@ public class SearchSourceBuilderTests extends ESTestCase { String restContent = " { \"_source\": false}"; try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser), - searchRequestParsers.aggParsers, searchRequestParsers.suggesters); + searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); assertArrayEquals(new String[]{}, searchSourceBuilder.fetchSource().excludes()); assertArrayEquals(new String[]{}, searchSourceBuilder.fetchSource().includes()); assertFalse(searchSourceBuilder.fetchSource().fetchSource()); @@ -519,7 +515,7 @@ public class SearchSourceBuilderTests extends ESTestCase { String restContent = " { \"sort\": \"foo\"}"; try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser), - searchRequestParsers.aggParsers, searchRequestParsers.suggesters); + searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); assertEquals(1, searchSourceBuilder.sorts().size()); assertEquals(new FieldSortBuilder("foo"), searchSourceBuilder.sorts().get(0)); } @@ -535,7 +531,7 @@ public class SearchSourceBuilderTests extends ESTestCase { " ]}"; try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser), - searchRequestParsers.aggParsers, searchRequestParsers.suggesters); + searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); assertEquals(5, searchSourceBuilder.sorts().size()); assertEquals(new FieldSortBuilder("post_date"), searchSourceBuilder.sorts().get(0)); assertEquals(new FieldSortBuilder("user"), searchSourceBuilder.sorts().get(1)); @@ -559,7 +555,7 @@ public class SearchSourceBuilderTests extends ESTestCase { "}\n"; try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser), - searchRequestParsers.aggParsers, searchRequestParsers.suggesters); + searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); assertEquals(1, searchSourceBuilder.aggregations().count()); } } @@ -575,7 +571,7 @@ public class SearchSourceBuilderTests extends ESTestCase { "}\n"; try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser), - searchRequestParsers.aggParsers, searchRequestParsers.suggesters); + searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); assertEquals(1, searchSourceBuilder.aggregations().count()); } } @@ -601,7 +597,7 @@ public class SearchSourceBuilderTests extends ESTestCase { "}\n"; try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser), - searchRequestParsers.aggParsers, searchRequestParsers.suggesters); + searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); assertEquals(1, searchSourceBuilder.rescores().size()); assertEquals(new QueryRescorerBuilder(QueryBuilders.matchQuery("content", "baz")).windowSize(50), searchSourceBuilder.rescores().get(0)); @@ -624,7 +620,7 @@ public class SearchSourceBuilderTests extends ESTestCase { "}\n"; try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser), - searchRequestParsers.aggParsers, searchRequestParsers.suggesters); + searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); assertEquals(1, searchSourceBuilder.rescores().size()); assertEquals(new QueryRescorerBuilder(QueryBuilders.matchQuery("content", "baz")).windowSize(50), searchSourceBuilder.rescores().get(0)); @@ -637,7 +633,7 @@ public class SearchSourceBuilderTests extends ESTestCase { final String query = "{ \"query\": { \"match_all\": {}}, \"timeout\": \"" + timeout + "\"}"; try (XContentParser parser = XContentFactory.xContent(query).createParser(query)) { final SearchSourceBuilder builder = SearchSourceBuilder.fromXContent(createParseContext(parser), - searchRequestParsers.aggParsers, searchRequestParsers.suggesters); + searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers); assertThat(builder.timeout(), equalTo(TimeValue.parseTimeValue(timeout, null, "timeout"))); } } @@ -650,7 +646,7 @@ public class SearchSourceBuilderTests extends ESTestCase { expectThrows( ElasticsearchParseException.class, () -> SearchSourceBuilder.fromXContent(createParseContext(parser), - searchRequestParsers.aggParsers, searchRequestParsers.suggesters)); + searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers)); assertThat(e, hasToString(containsString("unit is missing or unrecognized"))); } } diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java index f8da0312f88..87965365fd1 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java @@ -26,14 +26,18 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.SearchExtBuilder; +import org.elasticsearch.search.SearchExtParser; import org.elasticsearch.search.SearchHitField; -import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.InternalSearchHitField; import org.elasticsearch.search.internal.SearchContext; @@ -48,21 +52,27 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; import static org.elasticsearch.client.Requests.indexRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.CoreMatchers.equalTo; -@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) +@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2) public class FetchSubPhasePluginIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { return Collections.singletonList(FetchTermVectorsPlugin.class); } + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + @SuppressWarnings("unchecked") public void testPlugin() throws Exception { client().admin() .indices() @@ -84,12 +94,11 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase { client().admin().indices().prepareRefresh().execute().actionGet(); - XContentBuilder extSource = jsonBuilder().startObject() - .field("term_vectors_fetch", "test") - .endObject(); - SearchResponse response = client().prepareSearch().setSource(new SearchSourceBuilder().ext(extSource)).get(); + SearchResponse response = client().prepareSearch().setSource(new SearchSourceBuilder() + .ext(Collections.singletonList(new TermVectorsFetchBuilder("test")))).get(); assertSearchResponse(response); - assertThat(((Map) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("i"), equalTo(2)); + assertThat(((Map) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("i"), + equalTo(2)); assertThat(((Map) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("am"), equalTo(2)); assertThat(((Map) response.getHits().getAt(0).field("term_vectors_fetch").getValues().get(0)).get("sam"), @@ -101,46 +110,35 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase { public List getFetchSubPhases(FetchPhaseConstructionContext context) { return singletonList(new TermVectorsFetchSubPhase()); } + + @Override + public List> getSearchExts() { + return Collections.singletonList(new SearchExtSpec<>(TermVectorsFetchSubPhase.NAME, + TermVectorsFetchBuilder::new, TermVectorsFetchParser.INSTANCE)); + } } public static final class TermVectorsFetchSubPhase implements FetchSubPhase { - - public static final ContextFactory CONTEXT_FACTORY = new ContextFactory() { - - @Override - public String getName() { - return NAMES[0]; - } - - @Override - public TermVectorsFetchContext newContextInstance() { - return new TermVectorsFetchContext(); - } - }; - - public static final String[] NAMES = {"term_vectors_fetch"}; - - @Override - public Map parseElements() { - return singletonMap("term_vectors_fetch", new TermVectorsFetchParseElement()); - } + private static final String NAME = "term_vectors_fetch"; @Override public void hitExecute(SearchContext context, HitContext hitContext) { - if (context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded() == false) { + TermVectorsFetchBuilder fetchSubPhaseBuilder = (TermVectorsFetchBuilder)context.getSearchExt(NAME); + if (fetchSubPhaseBuilder == null) { return; } - String field = context.getFetchSubPhaseContext(CONTEXT_FACTORY).getField(); - + String field = fetchSubPhaseBuilder.getField(); if (hitContext.hit().fieldsOrNull() == null) { hitContext.hit().fields(new HashMap<>()); } - SearchHitField hitField = hitContext.hit().fields().get(NAMES[0]); + SearchHitField hitField = hitContext.hit().fields().get(NAME); if (hitField == null) { - hitField = new InternalSearchHitField(NAMES[0], new ArrayList<>(1)); - hitContext.hit().fields().put(NAMES[0], hitField); + hitField = new InternalSearchHitField(NAME, new ArrayList<>(1)); + hitContext.hit().fields().put(NAME, hitField); } - TermVectorsResponse termVector = TermVectorsService.getTermVectors(context.indexShard(), new TermVectorsRequest(context.indexShard().shardId().getIndex().getName(), hitContext.hit().type(), hitContext.hit().id())); + TermVectorsRequest termVectorsRequest = new TermVectorsRequest(context.indexShard().shardId().getIndex().getName(), + hitContext.hit().type(), hitContext.hit().id()); + TermVectorsResponse termVector = TermVectorsService.getTermVectors(context.indexShard(), termVectorsRequest); try { Map tv = new HashMap<>(); TermsEnum terms = termVector.getFields().terms(field).iterator(); @@ -155,36 +153,74 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase { } } - public static class TermVectorsFetchParseElement implements SearchParseElement { + public static final class TermVectorsFetchParser implements SearchExtParser { + + private static final TermVectorsFetchParser INSTANCE = new TermVectorsFetchParser(); + + private TermVectorsFetchParser() { + } @Override - public void parse(XContentParser parser, SearchContext context) throws Exception { - TermVectorsFetchContext fetchSubPhaseContext = context.getFetchSubPhaseContext(TermVectorsFetchSubPhase.CONTEXT_FACTORY); - // this is to make sure that the SubFetchPhase knows it should execute - fetchSubPhaseContext.setHitExecutionNeeded(true); + public TermVectorsFetchBuilder fromXContent(XContentParser parser) throws IOException { + String field; XContentParser.Token token = parser.currentToken(); if (token == XContentParser.Token.VALUE_STRING) { - String fieldName = parser.text(); - fetchSubPhaseContext.setField(fieldName); + field = parser.text(); } else { - throw new IllegalStateException("Expected a VALUE_STRING but got " + token); + throw new ParsingException(parser.getTokenLocation(), "Expected a VALUE_STRING but got " + token); } + if (field == null) { + throw new ParsingException(parser.getTokenLocation(), "no fields specified for " + TermVectorsFetchSubPhase.NAME); + } + return new TermVectorsFetchBuilder(field); } } - public static class TermVectorsFetchContext extends FetchSubPhaseContext { + public static final class TermVectorsFetchBuilder extends SearchExtBuilder { + private final String field; - private String field = null; - - public TermVectorsFetchContext() { + public TermVectorsFetchBuilder(String field) { + this.field = field; } - public void setField(String field) { - this.field = field; + public TermVectorsFetchBuilder(StreamInput in) throws IOException { + this.field = in.readString(); } public String getField() { return field; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TermVectorsFetchBuilder that = (TermVectorsFetchBuilder) o; + return Objects.equals(field, that.field); + } + + @Override + public int hashCode() { + return Objects.hash(field); + } + + @Override + public String getWriteableName() { + return TermVectorsFetchSubPhase.NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(field); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.field(TermVectorsFetchSubPhase.NAME, field); + } } } diff --git a/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java b/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java index c2e99b09dc3..eb37299306c 100644 --- a/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java +++ b/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java @@ -34,12 +34,14 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchRequestTests; +import org.elasticsearch.search.fetch.FetchSubPhasePluginIT; import org.elasticsearch.test.ESTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static java.util.Collections.emptyList; @@ -56,7 +58,8 @@ public class ShardSearchTransportRequestTests extends ESTestCase { bindMapperExtension(); } }; - SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()) { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, + Collections.singletonList(new FetchSubPhasePluginIT.FetchTermVectorsPlugin())) { @Override protected void configureSearch() { // Skip me diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java index 473e287c9a2..9df5a29b9c7 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java @@ -32,14 +32,11 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchRequestParsers; -import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -86,7 +83,7 @@ public class TransportSearchTemplateAction extends HandledTransportAction() { diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java index 44914e140b1..8b290d4c9c4 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java @@ -37,9 +37,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.SearchRequestParsers; -import org.elasticsearch.search.aggregations.AggregatorParsers; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -160,8 +158,7 @@ public class TransportMultiPercolateAction extends HandledTransportAction subPhaseContexts = new HashMap<>(); + private final Map searchExtBuilders = new HashMap<>(); public TestSearchContext(ThreadPool threadPool, BigArrays bigArrays, ScriptService scriptService, IndexService indexService) { super(ParseFieldMatcher.STRICT); @@ -197,12 +197,13 @@ public class TestSearchContext extends SearchContext { } @Override - public SubPhaseContext getFetchSubPhaseContext(FetchSubPhase.ContextFactory contextFactory) { - String subPhaseName = contextFactory.getName(); - if (subPhaseContexts.get(subPhaseName) == null) { - subPhaseContexts.put(subPhaseName, contextFactory.newContextInstance()); - } - return (SubPhaseContext) subPhaseContexts.get(subPhaseName); + public void addSearchExt(SearchExtBuilder searchExtBuilder) { + searchExtBuilders.put(searchExtBuilder.getWriteableName(), searchExtBuilder); + } + + @Override + public SearchExtBuilder getSearchExt(String name) { + return searchExtBuilders.get(name); } @Override @@ -262,6 +263,16 @@ public class TestSearchContext extends SearchContext { return null; } + @Override + public DocValueFieldsContext docValueFieldsContext() { + return null; + } + + @Override + public SearchContext docValueFieldsContext(DocValueFieldsContext docValueFieldsContext) { + return null; + } + @Override public ContextIndexSearcher searcher() { return searcher;