honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding method to build SuggestionContext to PhraseSuggestionBuilder 

This adds a build method for the SuggestionContext to the PhraseSuggestionBuilder
and another one that creates the SuggestionSearchContext to the top level
SuggestBuilder. Also adding tests that make sure the current way of parsing
xContent to a SuggestionContext is reflected in the output the builders create.
This commit is contained in:
Christoph Büscher 2016-02-02 15:48:27 +01:00
parent 421ed1228b
commit 9e0f6e3f9c
42 changed files with 1135 additions and 658 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java
View File

@ -142,8 +142,7 @@ public class TransportSuggestAction extends TransportBroadcastAction<SuggestRequ
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new IllegalArgumentException("suggest content missing");
}
final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.mapperService(),
indexService.fieldData(), request.shardId().getIndexName(), request.shardId().id());
final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.newQueryShardContext());
final Suggest result = suggestPhase.execute(context, searcher.searcher());
return new ShardSuggestResponse(request.shardId(), result);
}

2
core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
View File

@ -39,7 +39,7 @@ import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
import org.elasticsearch.search.rescore.RescoreBuilder;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.SmoothingModel;
import org.elasticsearch.tasks.Task;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;

2
core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
View File

@ -38,7 +38,7 @@ import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
import org.elasticsearch.search.rescore.RescoreBuilder;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.SmoothingModel;
import org.elasticsearch.tasks.Task;
import org.joda.time.ReadableInstant;

4
core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java
View File

@ -19,12 +19,12 @@
package org.elasticsearch.indices.query;
import java.util.Map;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryParser;
import java.util.Map;
public class IndicesQueriesRegistry extends AbstractComponent {
private Map<String, QueryParser<?>> queryParsers;

4
core/src/main/java/org/elasticsearch/search/SearchService.java
View File

@ -751,7 +751,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
if (source.rescores() != null) {
try {
for (RescoreBuilder<?> rescore : source.rescores()) {
context.addRescore(rescore.build(context.getQueryShardContext()));
context.addRescore(rescore.build(queryShardContext));
}
} catch (IOException e) {
throw new SearchContextException(context, "failed to create RescoreSearchContext", e);
@ -776,7 +776,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
if (source.highlighter() != null) {
HighlightBuilder highlightBuilder = source.highlighter();
try {
context.highlight(highlightBuilder.build(context.getQueryShardContext()));
context.highlight(highlightBuilder.build(queryShardContext));
} catch (IOException e) {
throw new SearchContextException(context, "failed to create SearchContextHighlighter", e);
}

18
core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java
View File

@ -26,9 +26,12 @@ import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import java.io.IOException;
import java.util.ArrayList;
@ -137,6 +140,21 @@ public class SuggestBuilder extends ToXContentToBytes implements Writeable<Sugge
return suggestBuilder;
}
public SuggestionSearchContext build(QueryShardContext context) throws IOException {
SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext();
for (SuggestionBuilder<?> suggestionBuilder : suggestions) {
SuggestionContext suggestionContext = suggestionBuilder.build(context);
if (suggestionContext.getText() == null) {
if (globalText == null) {
throw new IllegalArgumentException("The required text option is missing");
}
suggestionContext.setText(BytesRefs.toBytesRef(globalText));
}
suggestionSearchContext.addSuggestion(suggestionBuilder.name(), suggestionContext);
}
return suggestionSearchContext;
}
@Override
public SuggestBuilder readFrom(StreamInput in) throws IOException {
final SuggestBuilder builder = new SuggestBuilder();

5
core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java
View File

@ -19,12 +19,11 @@
package org.elasticsearch.search.suggest;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryShardContext;
import java.io.IOException;
public interface SuggestContextParser {
SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService indexFieldDataService) throws IOException;
SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException;
}

26
core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java
View File

@ -21,8 +21,8 @@ package org.elasticsearch.search.suggest;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
@ -44,14 +44,13 @@ public final class SuggestParseElement implements SearchParseElement {
@Override
public void parse(XContentParser parser, SearchContext context) throws Exception {
SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.fieldData(),
context.shardTarget().index(), context.shardTarget().shardId());
SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.getQueryShardContext());
context.suggest(suggestionSearchContext);
}
public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService,
String index, int shardId) throws IOException {
public SuggestionSearchContext parseInternal(XContentParser parser, QueryShardContext shardContext) throws IOException {
SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext();
MapperService mapperService = shardContext.getMapperService();
BytesRef globalText = null;
String fieldName = null;
@ -95,10 +94,20 @@ public final class SuggestParseElement implements SearchParseElement {
throw new IllegalArgumentException("Suggester[" + fieldName + "] not supported");
}
final SuggestContextParser contextParser = suggesters.get(fieldName).getContextParser();
suggestionContext = contextParser.parse(parser, mapperService, fieldDataService);
suggestionContext = contextParser.parse(parser, shardContext);
}
}
if (suggestionContext != null) {
if (suggestText != null) {
suggestionContext.setText(suggestText);
}
if (prefix != null) {
suggestionContext.setPrefix(prefix);
}
if (regex != null) {
suggestionContext.setRegex(regex);
}
if (suggestText != null && prefix == null) {
suggestionContext.setPrefix(suggestText);
suggestionContext.setText(suggestText);
@ -110,6 +119,8 @@ public final class SuggestParseElement implements SearchParseElement {
suggestionContext.setText(regex);
}
suggestionContexts.put(suggestionName, suggestionContext);
} else {
throw new IllegalArgumentException("suggestion context could not be parsed correctly");
}
}
}
@ -117,9 +128,6 @@ public final class SuggestParseElement implements SearchParseElement {
for (Map.Entry<String, SuggestionContext> entry : suggestionContexts.entrySet()) {
String suggestionName = entry.getKey();
SuggestionContext suggestionContext = entry.getValue();
suggestionContext.setShard(shardId);
suggestionContext.setIndex(index);
SuggestUtils.verifySuggestion(mapperService, globalText, suggestionContext);
suggestionSearchContext.addSuggestion(suggestionName, suggestionContext);
}

3
core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java
View File

@ -271,10 +271,10 @@ public final class SuggestUtils {
return false;
}
return true;
}
public static void verifySuggestion(MapperService mapperService, BytesRef globalText, SuggestionContext suggestion) {
// Verify options and set defaults
if (suggestion.getField() == null) {
@ -294,7 +294,6 @@ public final class SuggestUtils {
}
}
public static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) {
if (analyzer instanceof NamedAnalyzer) {
analyzer = ((NamedAnalyzer)analyzer).analyzer();

18
core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java
View File

@ -20,8 +20,6 @@ package org.elasticsearch.search.suggest;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.util.ExtensionPoint;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.suggest.completion.CompletionSuggester;
import org.elasticsearch.search.suggest.phrase.PhraseSuggester;
import org.elasticsearch.search.suggest.term.TermSuggester;
@ -42,21 +40,17 @@ public final class Suggesters extends ExtensionPoint.ClassMap<Suggester> {
this(Collections.emptyMap());
}
@Inject
public Suggesters(Map<String, Suggester> suggesters) {
super("suggester", Suggester.class, new HashSet<>(Arrays.asList("phrase", "term", "completion")), Suggesters.class, SuggestParseElement.class, SuggestPhase.class);
this.parsers = Collections.unmodifiableMap(suggesters);
this.parsers = Collections.unmodifiableMap(addBuildIns(suggesters));
}
@Inject
public Suggesters(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) {
this(addBuildIns(suggesters, scriptService, indexServices));
}
private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) {
private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters) {
final Map<String, Suggester> map = new HashMap<>();
map.put("phrase", new PhraseSuggester(scriptService, indexServices));
map.put("term", new TermSuggester());
map.put("completion", new CompletionSuggester());
map.put("phrase", PhraseSuggester.PROTOTYPE);
map.put("term", TermSuggester.PROTOTYPE);
map.put("completion", CompletionSuggester.PROTOTYPE);
map.putAll(suggesters);
return map;
}

55
core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
View File

@ -19,15 +19,20 @@
package org.elasticsearch.search.suggest;
import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.action.support.ToXContentToBytes;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import java.io.IOException;
import java.util.Objects;
@ -192,6 +197,56 @@ public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends
protected abstract SuggestionBuilder<T> innerFromXContent(QueryParseContext parseContext, String name) throws IOException;
protected abstract SuggestionContext build(QueryShardContext context) throws IOException;
/**
* Transfers the text, prefix, regex, analyzer, fieldname, size and shard size settings from the
* original {@link SuggestionBuilder} to the target {@link SuggestionContext}
*/
protected void populateCommonFields(MapperService mapperService,
SuggestionSearchContext.SuggestionContext suggestionContext) throws IOException {
if (analyzer != null) {
Analyzer luceneAnalyzer = mapperService.analysisService().analyzer(analyzer);
if (luceneAnalyzer == null) {
throw new IllegalArgumentException("Analyzer [" + luceneAnalyzer + "] doesn't exists");
}
suggestionContext.setAnalyzer(luceneAnalyzer);
}
if (fieldname != null) {
suggestionContext.setField(fieldname);
}
if (size != null) {
suggestionContext.setSize(size);
}
if (shardSize != null) {
suggestionContext.setShardSize(shardSize);
} else {
// if no shard size is set in builder, use size (or at least 5)
suggestionContext.setShardSize(Math.max(suggestionContext.getSize(), 5));
}
if (text != null) {
suggestionContext.setText(BytesRefs.toBytesRef(text));
}
if (prefix != null) {
suggestionContext.setPrefix(BytesRefs.toBytesRef(prefix));
}
if (regex != null) {
suggestionContext.setRegex(BytesRefs.toBytesRef(regex));
}
if (text != null && prefix == null) {
suggestionContext.setPrefix(BytesRefs.toBytesRef(text));
} else if (text == null && prefix != null) {
suggestionContext.setText(BytesRefs.toBytesRef(prefix));
} else if (text == null && regex != null) {
suggestionContext.setText(BytesRefs.toBytesRef(regex));
}
}
private String getSuggesterName() {
//default impl returns the same as writeable name, but we keep the distinction between the two just to make sure
return getWriteableName();

41
core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.search.suggest;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.query.QueryShardContext;
import java.util.LinkedHashMap;
import java.util.Map;
@ -36,20 +37,24 @@ public class SuggestionSearchContext {
public Map<String, SuggestionContext> suggestions() {
return suggestions;
}
public static class SuggestionContext {
public abstract static class SuggestionContext {
private BytesRef text;
private BytesRef prefix;
private BytesRef regex;
private final Suggester suggester;
private String field;
private Analyzer analyzer;
private int size = 5;
private int shardSize = -1;
private int shardId;
private String index;
private QueryShardContext shardContext;
private Suggester<?> suggester;
protected SuggestionContext(Suggester<?> suggester, QueryShardContext shardContext) {
this.suggester = suggester;
this.shardContext = shardContext;
}
public BytesRef getText() {
return text;
}
@ -74,12 +79,8 @@ public class SuggestionSearchContext {
this.regex = regex;
}
public SuggestionContext(Suggester suggester) {
this.suggester = suggester;
}
public Suggester<SuggestionContext> getSuggester() {
return this.suggester;
return ((Suggester<SuggestionContext>) suggester);
}
public Analyzer getAnalyzer() {
@ -119,21 +120,9 @@ public class SuggestionSearchContext {
}
this.shardSize = shardSize;
}
public void setShard(int shardId) {
this.shardId = shardId;
}
public void setIndex(String index) {
this.index = index;
}
public String getIndex() {
return index;
}
public int getShard() {
return shardId;
public QueryShardContext getShardContext() {
return this.shardContext;
}
}

8
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java
View File

@ -20,17 +20,16 @@ package org.elasticsearch.search.suggest.completion;
import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.RegexpFlag;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils.Fields;
@ -135,8 +134,9 @@ public class CompletionSuggestParser implements SuggestContextParser {
}
@Override
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester, mapperService, fieldDataService);
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException {
MapperService mapperService = shardContext.getMapperService();
final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(shardContext);
final ContextAndSuggest contextAndSuggest = new ContextAndSuggest(mapperService);
TLP_PARSER.parse(parser, suggestion, contextAndSuggest);
final XContentParser contextParser = contextAndSuggest.contextParser;

10
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
View File

@ -34,7 +34,9 @@ import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.Suggester;
@ -51,6 +53,8 @@ import java.util.Set;
public class CompletionSuggester extends Suggester<CompletionSuggestionContext> {
public static final CompletionSuggester PROTOTYPE = new CompletionSuggester();
@Override
public SuggestContextParser getContextParser() {
return new CompletionSuggestParser(this);
@ -86,9 +90,11 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
final LeafReaderContext subReaderContext = leaves.get(readerIndex);
final int subDocId = suggestDoc.doc - subReaderContext.docBase;
for (String field : payloadFields) {
MappedFieldType payloadFieldType = suggestionContext.getMapperService().fullName(field);
MapperService mapperService = suggestionContext.getShardContext().getMapperService();
MappedFieldType payloadFieldType = mapperService.fullName(field);
if (payloadFieldType != null) {
final AtomicFieldData data = suggestionContext.getIndexFieldDataService().getForField(payloadFieldType)
QueryShardContext shardContext = suggestionContext.getShardContext();
final AtomicFieldData data = shardContext.getForField(payloadFieldType)
.load(subReaderContext);
final ScriptDocValues scriptValues = data.getScriptValues();
scriptValues.setNextDocId(subDocId);

9
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
View File

@ -28,8 +28,10 @@ import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.RegexpFlag;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
import org.elasticsearch.search.suggest.completion.context.GeoQueryContext;
@ -372,9 +374,16 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
@Override
protected CompletionSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String name) throws IOException {
// NORELEASE
return new CompletionSuggestionBuilder(name);
}
@Override
protected SuggestionContext build(QueryShardContext context) throws IOException {
// NORELEASE
throw new UnsupportedOperationException();
}
@Override
public String getWriteableName() {
return SUGGESTION_NAME;

25
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java
View File

@ -20,10 +20,8 @@ package org.elasticsearch.search.suggest.completion;
import org.apache.lucene.search.suggest.document.CompletionQuery;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.SuggestionSearchContext;
import org.elasticsearch.search.suggest.completion.context.ContextMapping;
import org.elasticsearch.search.suggest.completion.context.ContextMappings;
@ -39,20 +37,16 @@ import java.util.Set;
*/
public class CompletionSuggestionContext extends SuggestionSearchContext.SuggestionContext {
protected CompletionSuggestionContext(QueryShardContext shardContext) {
super(CompletionSuggester.PROTOTYPE, shardContext);
}
private CompletionFieldMapper.CompletionFieldType fieldType;
private CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptionsBuilder;
private CompletionSuggestionBuilder.RegexOptionsBuilder regexOptionsBuilder;
private Map<String, List<ContextMapping.QueryContext>> queryContexts = Collections.emptyMap();
private final MapperService mapperService;
private final IndexFieldDataService indexFieldDataService;
private Set<String> payloadFields = Collections.emptySet();
CompletionSuggestionContext(Suggester suggester, MapperService mapperService, IndexFieldDataService indexFieldDataService) {
super(suggester);
this.indexFieldDataService = indexFieldDataService;
this.mapperService = mapperService;
}
CompletionFieldMapper.CompletionFieldType getFieldType() {
return this.fieldType;
}
@ -73,15 +67,6 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest
this.queryContexts = queryContexts;
}
MapperService getMapperService() {
return mapperService;
}
IndexFieldDataService getIndexFieldDataService() {
return indexFieldDataService;
}
void setPayloadFields(Set<String> fields) {
this.payloadFields = fields;
}

4
core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
View File

@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.CandidateGenerator;
@ -349,8 +348,7 @@ public final class DirectCandidateGeneratorBuilder
return replaceField(tmpFieldName.iterator().next(), tempGenerator);
}
public PhraseSuggestionContext.DirectCandidateGenerator build(QueryShardContext context) throws IOException {
MapperService mapperService = context.getMapperService();
public PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException {
PhraseSuggestionContext.DirectCandidateGenerator generator = new PhraseSuggestionContext.DirectCandidateGenerator();
generator.setField(this.field);
transferIfNotNull(this.size, generator::size);

126
core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java Normal file
View File

@ -0,0 +1,126 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
import java.io.IOException;
import java.util.Objects;
/**
* An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive
* smoothing</a> model.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public final class Laplace extends SmoothingModel {
private double alpha = DEFAULT_LAPLACE_ALPHA;
private static final String NAME = "laplace";
private static final ParseField ALPHA_FIELD = new ParseField("alpha");
static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Default alpha parameter for laplace smoothing
*/
public static final double DEFAULT_LAPLACE_ALPHA = 0.5;
public static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA);
/**
* Creates a Laplace smoothing model.
*
*/
public Laplace(double alpha) {
this.alpha = alpha;
}
/**
* @return the laplace model alpha parameter
*/
public double getAlpha() {
return this.alpha;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(ALPHA_FIELD.getPreferredName(), alpha);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(alpha);
}
@Override
public SmoothingModel readFrom(StreamInput in) throws IOException {
return new Laplace(in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
Laplace otherModel = (Laplace) other;
return Objects.equals(alpha, otherModel.alpha);
}
@Override
protected final int doHashCode() {
return Objects.hash(alpha);
}
@Override
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double alpha = DEFAULT_LAPLACE_ALPHA;
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
}
if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) {
alpha = parser.doubleValue();
}
}
return new Laplace(alpha);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-> new LaplaceScorer(reader, terms, field, realWordLikelyhood, separator, alpha);
}
}

176
core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java Normal file
View File

@ -0,0 +1,176 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
import java.io.IOException;
import java.util.Objects;
/**
* Linear interpolation smoothing model.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public final class LinearInterpolation extends SmoothingModel {
private static final String NAME = "linear";
public static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1);
private final double trigramLambda;
private final double bigramLambda;
private final double unigramLambda;
static final ParseField PARSE_FIELD = new ParseField(NAME);
private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda");
private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda");
private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda");
/**
* Creates a linear interpolation smoothing model.
*
* Note: the lambdas must sum up to one.
*
* @param trigramLambda
* the trigram lambda
* @param bigramLambda
* the bigram lambda
* @param unigramLambda
* the unigram lambda
*/
public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) {
double sum = trigramLambda + bigramLambda + unigramLambda;
if (Math.abs(sum - 1.0) > 0.001) {
throw new IllegalArgumentException("linear smoothing lambdas must sum to 1");
}
this.trigramLambda = trigramLambda;
this.bigramLambda = bigramLambda;
this.unigramLambda = unigramLambda;
}
public double getTrigramLambda() {
return this.trigramLambda;
}
public double getBigramLambda() {
return this.bigramLambda;
}
public double getUnigramLambda() {
return this.unigramLambda;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda);
builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda);
builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(trigramLambda);
out.writeDouble(bigramLambda);
out.writeDouble(unigramLambda);
}
@Override
public LinearInterpolation readFrom(StreamInput in) throws IOException {
return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
final LinearInterpolation otherModel = (LinearInterpolation) other;
return Objects.equals(trigramLambda, otherModel.trigramLambda) &&
Objects.equals(bigramLambda, otherModel.bigramLambda) &&
Objects.equals(unigramLambda, otherModel.unigramLambda);
}
@Override
protected final int doHashCode() {
return Objects.hash(trigramLambda, bigramLambda, unigramLambda);
}
@Override
public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double trigramLambda = 0.0;
double bigramLambda = 0.0;
double unigramLambda = 0.0;
ParseFieldMatcher matcher = parseContext.parseFieldMatcher();
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
if (matcher.match(fieldName, TRIGRAM_FIELD)) {
trigramLambda = parser.doubleValue();
if (trigramLambda < 0) {
throw new IllegalArgumentException("trigram_lambda must be positive");
}
} else if (matcher.match(fieldName, BIGRAM_FIELD)) {
bigramLambda = parser.doubleValue();
if (bigramLambda < 0) {
throw new IllegalArgumentException("bigram_lambda must be positive");
}
} else if (matcher.match(fieldName, UNIGRAM_FIELD)) {
unigramLambda = parser.doubleValue();
if (unigramLambda < 0) {
throw new IllegalArgumentException("unigram_lambda must be positive");
}
} else {
throw new IllegalArgumentException(
"suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(),
"[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) ->
new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda,
unigramLambda);
}
}

16
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java
View File

@ -26,17 +26,16 @@ import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.Template;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.SuggestionSearchContext;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
import java.io.IOException;
@ -51,8 +50,10 @@ public final class PhraseSuggestParser implements SuggestContextParser {
}
@Override
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
PhraseSuggestionContext suggestion = new PhraseSuggestionContext(suggester);
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException {
MapperService mapperService = shardContext.getMapperService();
ScriptService scriptService = shardContext.getScriptService();
PhraseSuggestionContext suggestion = new PhraseSuggestionContext(shardContext);
ParseFieldMatcher parseFieldMatcher = mapperService.getIndexSettings().getParseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
@ -135,7 +136,7 @@ public final class PhraseSuggestParser implements SuggestContextParser {
throw new IllegalArgumentException("suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]");
}
Template template = Template.parse(parser, parseFieldMatcher);
CompiledScript compiledScript = suggester.scriptService().compile(template, ScriptContext.Standard.SEARCH, Collections.emptyMap());
CompiledScript compiledScript = scriptService.compile(template, ScriptContext.Standard.SEARCH, Collections.emptyMap());
suggestion.setCollateQueryScript(compiledScript);
} else if ("params".equals(fieldName)) {
suggestion.setCollateScriptParams(parser.map());
@ -199,9 +200,6 @@ public final class PhraseSuggestParser implements SuggestContextParser {
suggestion.addGenerator(generator);
}
}
return suggestion;
}

20
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
View File

@ -31,9 +31,7 @@ import org.apache.lucene.util.CharsRefBuilder;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.query.ParsedQuery;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.ScriptService;
@ -44,6 +42,7 @@ import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result;
import java.io.IOException;
@ -54,13 +53,8 @@ import java.util.Map;
public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
private final BytesRef SEPARATOR = new BytesRef(" ");
private static final String SUGGESTION_TEMPLATE_VAR_NAME = "suggestion";
private final ScriptService scriptService;
private final IndicesService indicesService;
public PhraseSuggester(ScriptService scriptService, IndicesService indicesService) {
this.scriptService = scriptService;
this.indicesService = indicesService;
}
public static final PhraseSuggester PROTOTYPE = new PhraseSuggester();
/*
* More Ideas:
@ -118,10 +112,10 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
// from the index for a correction, collateMatch is updated
final Map<String, Object> vars = suggestion.getCollateScriptParams();
vars.put(SUGGESTION_TEMPLATE_VAR_NAME, spare.toString());
ScriptService scriptService = suggestion.getShardContext().getScriptService();
final ExecutableScript executable = scriptService.executable(collateScript, vars);
final BytesReference querySource = (BytesReference) executable.run();
IndexService indexService = indicesService.indexService(suggestion.getIndex());
final ParsedQuery parsedQuery = indexService.newQueryShardContext().parse(querySource);
final ParsedQuery parsedQuery = suggestion.getShardContext().parse(querySource);
collateMatch = Lucene.exists(searcher, parsedQuery.query());
}
if (!collateMatch && !collatePrune) {
@ -145,15 +139,11 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
return response;
}
private PhraseSuggestion.Entry buildResultEntry(PhraseSuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) {
private PhraseSuggestion.Entry buildResultEntry(SuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) {
spare.copyUTF8Bytes(suggestion.getText());
return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore);
}
ScriptService scriptService() {
return scriptService;
}
@Override
public SuggestContextParser getContextParser() {
return new PhraseSuggestParser(this);

524
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
View File

@ -19,27 +19,32 @@
package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.Template;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -239,7 +244,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
/**
* Sets an explicit smoothing model used for this suggester. The default is
* {@link PhraseSuggestionBuilder.StupidBackoff}.
* {@link StupidBackoff}.
*/
public PhraseSuggestionBuilder smoothingModel(SmoothingModel model) {
this.model = model;
@ -254,6 +259,9 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
public PhraseSuggestionBuilder tokenLimit(int tokenLimit) {
if (tokenLimit <= 0) {
throw new IllegalArgumentException("token_limit must be >= 1");
}
this.tokenLimit = tokenLimit;
return this;
}
@ -389,413 +397,6 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
return builder;
}
/**
* Creates a new {@link DirectCandidateGeneratorBuilder}
*
* @param field
* the field this candidate generator operates on.
*/
public static DirectCandidateGeneratorBuilder candidateGenerator(String field) {
return new DirectCandidateGeneratorBuilder(field);
}
/**
* A "stupid-backoff" smoothing model simialr to <a
* href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's
* Backoff</a>. This model is used as the default if no model is configured.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public static final class StupidBackoff extends SmoothingModel {
/**
* Default discount parameter for {@link StupidBackoff} smoothing
*/
public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4;
public static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT);
private double discount = DEFAULT_BACKOFF_DISCOUNT;
private static final String NAME = "stupid_backoff";
private static final ParseField DISCOUNT_FIELD = new ParseField("discount");
private static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Creates a Stupid-Backoff smoothing model.
*
* @param discount
* the discount given to lower order ngrams if the higher order ngram doesn't exits
*/
public StupidBackoff(double discount) {
this.discount = discount;
}
/**
* @return the discount parameter of the model
*/
public double getDiscount() {
return this.discount;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(DISCOUNT_FIELD.getPreferredName(), discount);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(discount);
}
@Override
public StupidBackoff readFrom(StreamInput in) throws IOException {
return new StupidBackoff(in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
StupidBackoff otherModel = (StupidBackoff) other;
return Objects.equals(discount, otherModel.discount);
}
@Override
protected final int doHashCode() {
return Objects.hash(discount);
}
@Override
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double discount = DEFAULT_BACKOFF_DISCOUNT;
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
}
if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) {
discount = parser.doubleValue();
}
}
return new StupidBackoff(discount);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount);
}
}
/**
* An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive
* smoothing</a> model.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public static final class Laplace extends SmoothingModel {
private double alpha = DEFAULT_LAPLACE_ALPHA;
private static final String NAME = "laplace";
private static final ParseField ALPHA_FIELD = new ParseField("alpha");
private static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Default alpha parameter for laplace smoothing
*/
public static final double DEFAULT_LAPLACE_ALPHA = 0.5;
public static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA);
/**
* Creates a Laplace smoothing model.
*
*/
public Laplace(double alpha) {
this.alpha = alpha;
}
/**
* @return the laplace model alpha parameter
*/
public double getAlpha() {
return this.alpha;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(ALPHA_FIELD.getPreferredName(), alpha);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(alpha);
}
@Override
public SmoothingModel readFrom(StreamInput in) throws IOException {
return new Laplace(in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
Laplace otherModel = (Laplace) other;
return Objects.equals(alpha, otherModel.alpha);
}
@Override
protected final int doHashCode() {
return Objects.hash(alpha);
}
@Override
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double alpha = DEFAULT_LAPLACE_ALPHA;
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
}
if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) {
alpha = parser.doubleValue();
}
}
return new Laplace(alpha);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-> new LaplaceScorer(reader, terms, field, realWordLikelyhood, separator, alpha);
}
}
public static abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(getWriteableName());
innerToXContent(builder,params);
builder.endObject();
return builder;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
SmoothingModel other = (SmoothingModel) obj;
return doEquals(other);
}
public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
SmoothingModel model = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) {
model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) {
model = Laplace.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) {
model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext);
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(),
"[smoothing] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return model;
}
public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException;
@Override
public final int hashCode() {
/*
* Override hashCode here and forward to an abstract method to force extensions of this class to override hashCode in the same
* way that we force them to override equals. This also prevents false positives in CheckStyle's EqualsHashCode check.
*/
return doHashCode();
}
public abstract WordScorerFactory buildWordScorerFactory();
/**
* subtype specific implementation of "equals".
*/
protected abstract boolean doEquals(SmoothingModel other);
protected abstract int doHashCode();
protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
}
/**
* Linear interpolation smoothing model.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public static final class LinearInterpolation extends SmoothingModel {
private static final String NAME = "linear";
public static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1);
private final double trigramLambda;
private final double bigramLambda;
private final double unigramLambda;
private static final ParseField PARSE_FIELD = new ParseField(NAME);
private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda");
private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda");
private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda");
/**
* Creates a linear interpolation smoothing model.
*
* Note: the lambdas must sum up to one.
*
* @param trigramLambda
* the trigram lambda
* @param bigramLambda
* the bigram lambda
* @param unigramLambda
* the unigram lambda
*/
public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) {
double sum = trigramLambda + bigramLambda + unigramLambda;
if (Math.abs(sum - 1.0) > 0.001) {
throw new IllegalArgumentException("linear smoothing lambdas must sum to 1");
}
this.trigramLambda = trigramLambda;
this.bigramLambda = bigramLambda;
this.unigramLambda = unigramLambda;
}
public double getTrigramLambda() {
return this.trigramLambda;
}
public double getBigramLambda() {
return this.bigramLambda;
}
public double getUnigramLambda() {
return this.unigramLambda;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda);
builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda);
builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(trigramLambda);
out.writeDouble(bigramLambda);
out.writeDouble(unigramLambda);
}
@Override
public LinearInterpolation readFrom(StreamInput in) throws IOException {
return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
final LinearInterpolation otherModel = (LinearInterpolation) other;
return Objects.equals(trigramLambda, otherModel.trigramLambda) &&
Objects.equals(bigramLambda, otherModel.bigramLambda) &&
Objects.equals(unigramLambda, otherModel.unigramLambda);
}
@Override
protected final int doHashCode() {
return Objects.hash(trigramLambda, bigramLambda, unigramLambda);
}
@Override
public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double trigramLambda = 0.0;
double bigramLambda = 0.0;
double unigramLambda = 0.0;
ParseFieldMatcher matcher = parseContext.parseFieldMatcher();
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
if (matcher.match(fieldName, TRIGRAM_FIELD)) {
trigramLambda = parser.doubleValue();
if (trigramLambda < 0) {
throw new IllegalArgumentException("trigram_lambda must be positive");
}
} else if (matcher.match(fieldName, BIGRAM_FIELD)) {
bigramLambda = parser.doubleValue();
if (bigramLambda < 0) {
throw new IllegalArgumentException("bigram_lambda must be positive");
}
} else if (matcher.match(fieldName, UNIGRAM_FIELD)) {
unigramLambda = parser.doubleValue();
if (unigramLambda < 0) {
throw new IllegalArgumentException("unigram_lambda must be positive");
}
} else {
throw new IllegalArgumentException(
"suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(),
"[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) ->
new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda,
unigramLambda);
}
}
@Override
protected PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String suggestionName) throws IOException {
XContentParser parser = parseContext.parser();
@ -873,7 +474,6 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
"suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]");
}
Template template = Template.parse(parser, parseFieldMatcher);
// TODO remember to compile script in build() method
suggestion.collateQuery(template);
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS)) {
suggestion.collateParams(parser.map());
@ -898,6 +498,98 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
return suggestion;
}
@Override
public SuggestionContext build(QueryShardContext context) throws IOException {
PhraseSuggestionContext suggestionContext = new PhraseSuggestionContext(context);
MapperService mapperService = context.getMapperService();
populateCommonFields(mapperService, suggestionContext);
suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator));
suggestionContext.setRealWordErrorLikelihood(this.realWordErrorLikelihood);
suggestionContext.setConfidence(this.confidence);
suggestionContext.setMaxErrors(this.maxErrors);
suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator));
suggestionContext.setRequireUnigram(this.forceUnigrams);
suggestionContext.setTokenLimit(this.tokenLimit);
suggestionContext.setPreTag(BytesRefs.toBytesRef(this.preTag));
suggestionContext.setPostTag(BytesRefs.toBytesRef(this.postTag));
if (this.gramSize != null) {
suggestionContext.setGramSize(this.gramSize);
}
for (List<CandidateGenerator> candidateGenerators : this.generators.values()) {
for (CandidateGenerator candidateGenerator : candidateGenerators) {
suggestionContext.addGenerator(candidateGenerator.build(mapperService));
}
}
if (this.model != null) {
suggestionContext.setModel(this.model.buildWordScorerFactory());
}
if (this.collateQuery != null) {
CompiledScript compiledScript = context.getScriptService().compile(this.collateQuery, ScriptContext.Standard.SEARCH,
Collections.emptyMap());
suggestionContext.setCollateQueryScript(compiledScript);
if (this.collateParams != null) {
suggestionContext.setCollateScriptParams(this.collateParams);
}
suggestionContext.setCollatePrune(this.collatePrune);
}
// TODO remove this when field is mandatory in builder ctor
if (suggestionContext.getField() == null) {
throw new IllegalArgumentException("The required field option is missing");
}
MappedFieldType fieldType = mapperService.fullName(suggestionContext.getField());
if (fieldType == null) {
throw new IllegalArgumentException("No mapping found for field [" + suggestionContext.getField() + "]");
} else if (suggestionContext.getAnalyzer() == null) {
// no analyzer name passed in, so try the field's analyzer, or the default analyzer
if (fieldType.searchAnalyzer() == null) {
suggestionContext.setAnalyzer(mapperService.searchAnalyzer());
} else {
suggestionContext.setAnalyzer(fieldType.searchAnalyzer());
}
}
if (suggestionContext.model() == null) {
suggestionContext.setModel(StupidBackoffScorer.FACTORY);
}
if (this.gramSize == null || suggestionContext.generators().isEmpty()) {
final ShingleTokenFilterFactory.Factory shingleFilterFactory = SuggestUtils
.getShingleFilterFactory(suggestionContext.getAnalyzer());
if (this.gramSize == null) {
// try to detect the shingle size
if (shingleFilterFactory != null) {
suggestionContext.setGramSize(shingleFilterFactory.getMaxShingleSize());
if (suggestionContext.getAnalyzer() == null && shingleFilterFactory.getMinShingleSize() > 1
&& !shingleFilterFactory.getOutputUnigrams()) {
throw new IllegalArgumentException("The default analyzer for field: [" + suggestionContext.getField()
+ "] doesn't emit unigrams. If this is intentional try to set the analyzer explicitly");
}
}
}
if (suggestionContext.generators().isEmpty()) {
if (shingleFilterFactory != null && shingleFilterFactory.getMinShingleSize() > 1
&& !shingleFilterFactory.getOutputUnigrams() && suggestionContext.getRequireUnigram()) {
throw new IllegalArgumentException("The default candidate generator for phrase suggest can't operate on field: ["
+ suggestionContext.getField() + "] since it doesn't emit unigrams. "
+ "If this is intentional try to set the candidate generator field explicitly");
}
// use a default generator on the same field
DirectCandidateGenerator generator = new DirectCandidateGenerator();
generator.setField(suggestionContext.getField());
suggestionContext.addGenerator(generator);
}
}
return suggestionContext;
}
private static void ensureNoSmoothing(PhraseSuggestionBuilder suggestion) {
if (suggestion.smoothingModel() != null) {
throw new IllegalArgumentException("only one smoothing model supported");
@ -1010,5 +702,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
String getType();
CandidateGenerator fromXContent(QueryParseContext parseContext) throws IOException;
PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException;
}
}

9
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
View File

@ -20,9 +20,9 @@ package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import java.util.ArrayList;
@ -54,8 +54,8 @@ class PhraseSuggestionContext extends SuggestionContext {
private Map<String, Object> collateScriptParams = new HashMap<>(1);
private WordScorer.WordScorerFactory scorer;
public PhraseSuggestionContext(Suggester<? extends PhraseSuggestionContext> suggester) {
super(suggester);
public PhraseSuggestionContext(QueryShardContext shardContext) {
super(PhraseSuggester.PROTOTYPE, shardContext);
}
public float maxErrors() {
@ -154,8 +154,6 @@ class PhraseSuggestionContext extends SuggestionContext {
public void postFilter(Analyzer postFilter) {
this.postFilter = postFilter;
}
}
public void setRequireUnigram(boolean requireUnigram) {
@ -213,5 +211,4 @@ class PhraseSuggestionContext extends SuggestionContext {
boolean collatePrune() {
return prune;
}
}

105
core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java Normal file
View File

@ -0,0 +1,105 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.suggest.phrase;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
import java.io.IOException;
public abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(getWriteableName());
innerToXContent(builder,params);
builder.endObject();
return builder;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
SmoothingModel other = (SmoothingModel) obj;
return doEquals(other);
}
@Override
public final int hashCode() {
/*
* Override hashCode here and forward to an abstract method to force
* extensions of this class to override hashCode in the same way that we
* force them to override equals. This also prevents false positives in
* CheckStyle's EqualsHashCode check.
*/
return doHashCode();
}
protected abstract int doHashCode();
public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
SmoothingModel model = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) {
model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) {
model = Laplace.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) {
model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext);
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(),
"[smoothing] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return model;
}
public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException;
public abstract WordScorerFactory buildWordScorerFactory();
/**
* subtype specific implementation of "equals".
*/
protected abstract boolean doEquals(SmoothingModel other);
protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
}

129
core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java Normal file
View File

@ -0,0 +1,129 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
import java.io.IOException;
import java.util.Objects;
/**
* A "stupid-backoff" smoothing model simialr to <a
* href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's
* Backoff</a>. This model is used as the default if no model is configured.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public final class StupidBackoff extends SmoothingModel {
/**
* Default discount parameter for {@link StupidBackoff} smoothing
*/
public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4;
public static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT);
private double discount = DEFAULT_BACKOFF_DISCOUNT;
private static final String NAME = "stupid_backoff";
private static final ParseField DISCOUNT_FIELD = new ParseField("discount");
static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Creates a Stupid-Backoff smoothing model.
*
* @param discount
* the discount given to lower order ngrams if the higher order ngram doesn't exits
*/
public StupidBackoff(double discount) {
this.discount = discount;
}
/**
* @return the discount parameter of the model
*/
public double getDiscount() {
return this.discount;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(DISCOUNT_FIELD.getPreferredName(), discount);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(discount);
}
@Override
public StupidBackoff readFrom(StreamInput in) throws IOException {
return new StupidBackoff(in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
StupidBackoff otherModel = (StupidBackoff) other;
return Objects.equals(discount, otherModel.discount);
}
@Override
protected final int doHashCode() {
return Objects.hash(discount);
}
@Override
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double discount = DEFAULT_BACKOFF_DISCOUNT;
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
}
if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) {
discount = parser.doubleValue();
}
}
return new StupidBackoff(discount);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount);
}
}

7
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java
View File

@ -20,8 +20,8 @@ package org.elasticsearch.search.suggest.term;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils;
@ -38,10 +38,11 @@ public final class TermSuggestParser implements SuggestContextParser {
}
@Override
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException {
MapperService mapperService = shardContext.getMapperService();
XContentParser.Token token;
String fieldName = null;
TermSuggestionContext suggestion = new TermSuggestionContext(suggester);
TermSuggestionContext suggestion = new TermSuggestionContext(shardContext);
DirectSpellcheckerSettings settings = suggestion.getDirectSpellCheckerSettings();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {

2
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
View File

@ -40,6 +40,8 @@ import java.util.List;
public final class TermSuggester extends Suggester<TermSuggestionContext> {
public static final TermSuggester PROTOTYPE = new TermSuggester();
@Override
public TermSuggestion innerExecute(String name, TermSuggestionContext suggestion, IndexSearcher searcher, CharsRefBuilder spare)
throws IOException {

8
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
View File

@ -26,7 +26,9 @@ import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import java.io.IOException;
import java.util.Locale;
@ -381,6 +383,12 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
return suggestion;
}
@Override
protected SuggestionContext build(QueryShardContext context) throws IOException {
// NORELEASE
throw new UnsupportedOperationException();
}
@Override
public String getWriteableName() {
return SUGGESTION_NAME;

7
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java
View File

@ -18,20 +18,19 @@
*/
package org.elasticsearch.search.suggest.term;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
final class TermSuggestionContext extends SuggestionContext {
private final DirectSpellcheckerSettings settings = new DirectSpellcheckerSettings();
public TermSuggestionContext(Suggester<? extends TermSuggestionContext> suggester) {
super(suggester);
public TermSuggestionContext(QueryShardContext shardContext) {
super(TermSuggester.PROTOTYPE, shardContext);
}
public DirectSpellcheckerSettings getDirectSpellCheckerSettings() {
return settings;
}
}

2
core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
View File

@ -63,8 +63,8 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.script.ScriptContextRegistry;
import org.elasticsearch.script.ScriptEngineRegistry;
import org.elasticsearch.script.ScriptEngineService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import org.elasticsearch.test.engine.MockEngineFactory;

164
core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.suggest;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
@ -31,19 +32,48 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperBuilders;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.StringFieldMapper;
import org.elasticsearch.index.mapper.core.StringFieldMapper.StringFieldType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptContextRegistry;
import org.elasticsearch.script.ScriptEngineRegistry;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptServiceTests.TestEngineService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import org.elasticsearch.watcher.ResourceWatcherService;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.function.Consumer;
import java.util.function.Supplier;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
@ -51,13 +81,35 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
private static final int NUMBER_OF_TESTBUILDERS = 20;
protected static NamedWriteableRegistry namedWriteableRegistry;
private static final Suggesters suggesters = new Suggesters(Collections.emptyMap(), null, null);
private static Suggesters suggesters;
private static ScriptService scriptService;
private static SuggestParseElement parseElement;
/**
* setup for the whole base test class
*/
@BeforeClass
public static void init() {
public static void init() throws IOException {
Path genericConfigFolder = createTempDir();
Settings baseSettings = settingsBuilder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put(Environment.PATH_CONF_SETTING.getKey(), genericConfigFolder)
.build();
Environment environment = new Environment(baseSettings);
ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry
.ScriptEngineRegistration(TestEngineService.class, TestEngineService.TYPES)));
ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
scriptService = new ScriptService(baseSettings, environment, Collections.singleton(new TestEngineService()),
new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
@Override
public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params) {
return new CompiledScript(ScriptType.INLINE, "mockName", "mocklang", script);
}
};
suggesters = new Suggesters(Collections.emptyMap());
parseElement = new SuggestParseElement(suggesters);
namedWriteableRegistry = new NamedWriteableRegistry();
namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, TermSuggestionBuilder.PROTOTYPE);
namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, PhraseSuggestionBuilder.PROTOTYPE);
@ -69,7 +121,6 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
namedWriteableRegistry = null;
}
/**
* Test serialization and deserialization of the suggestion builder
*/
@ -88,13 +139,13 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
*/
protected SB randomTestBuilder() {
SB randomSuggestion = randomSuggestionBuilder();
randomSuggestion.field(randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::text, randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::prefix, randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::regex, randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::field, randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::analyzer, randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::size, randomIntBetween(1, 20));
maybeSet(randomSuggestion::shardSize, randomInt(20));
maybeSet(randomSuggestion::shardSize, randomIntBetween(1, 20));
return randomSuggestion;
}
@ -137,7 +188,8 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
}
/**
* creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
* creates random suggestion builder, renders it to xContent and back to new
* instance that should be equal to original
*/
public void testFromXContent() throws IOException {
QueryParseContext context = new QueryParseContext(null);
@ -166,6 +218,97 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
}
}
/**
* parses random suggestion builder via old parseElement method and via
* build, comparing the results for equality
*/
public void testBuild() throws IOException {
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), Settings.EMPTY);
AnalysisService mockAnalysisService = new AnalysisService(idxSettings, Collections.emptyMap(), Collections.emptyMap(),
Collections.emptyMap(), Collections.emptyMap()) {
@Override
public NamedAnalyzer analyzer(String name) {
return new NamedAnalyzer(name, new WhitespaceAnalyzer());
}
};
MapperService mockMapperService = new MapperService(idxSettings, mockAnalysisService, null, new IndicesModule().getMapperRegistry(),
null) {
@Override
public MappedFieldType fullName(String fullName) {
return new StringFieldType();
}
};
QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, mockMapperService, null, scriptService,
null) {
@Override
public MappedFieldType fieldMapper(String name) {
StringFieldMapper.Builder builder = MapperBuilders.stringField(name);
return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType();
}
};
mockShardContext.setMapUnmappedFieldAsString(true);
for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
SuggestBuilder suggestBuilder = new SuggestBuilder();
SB suggestionBuilder = randomTestBuilder();
suggestBuilder.addSuggestion(suggestionBuilder);
if (suggestionBuilder.text() == null) {
// we either need suggestion text or global text
suggestBuilder.setText(randomAsciiOfLengthBetween(5, 50));
}
if (suggestionBuilder.text() != null && suggestionBuilder.prefix() != null) {
suggestionBuilder.prefix(null);
}
XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
if (randomBoolean()) {
xContentBuilder.prettyPrint();
}
suggestBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes());
parser.nextToken(); // set cursor to START_OBJECT
SuggestionSearchContext parsedSuggestionSearchContext = parseElement.parseInternal(parser, mockShardContext);
SuggestionSearchContext buildSuggestSearchContext = suggestBuilder.build(mockShardContext);
assertEquals(parsedSuggestionSearchContext.suggestions().size(), buildSuggestSearchContext.suggestions().size());
Iterator<Entry<String, SuggestionContext>> iterator = buildSuggestSearchContext.suggestions().entrySet().iterator();
for (Entry<String, SuggestionContext> entry : parsedSuggestionSearchContext.suggestions().entrySet()) {
Entry<String, SuggestionContext> other = iterator.next();
assertEquals(entry.getKey(), other.getKey());
SuggestionContext oldSchoolContext = entry.getValue();
SuggestionContext newSchoolContext = other.getValue();
assertNotSame(oldSchoolContext, newSchoolContext);
// deep comparison of analyzers is difficult here, but we check they are set or not set
if (oldSchoolContext.getAnalyzer() != null) {
assertNotNull(newSchoolContext.getAnalyzer());
} else {
assertNull(newSchoolContext.getAnalyzer());
}
assertEquals(oldSchoolContext.getField(), newSchoolContext.getField());
assertEquals(oldSchoolContext.getPrefix(), newSchoolContext.getPrefix());
assertEquals(oldSchoolContext.getRegex(), newSchoolContext.getRegex());
assertEquals(oldSchoolContext.getShardSize(), newSchoolContext.getShardSize());
assertEquals(oldSchoolContext.getSize(), newSchoolContext.getSize());
assertEquals(oldSchoolContext.getSuggester().getClass(), newSchoolContext.getSuggester().getClass());
assertEquals(oldSchoolContext.getText(), newSchoolContext.getText());
assertEquals(oldSchoolContext.getClass(), newSchoolContext.getClass());
assertSuggestionContext(oldSchoolContext, newSchoolContext);
}
}
}
/**
* compare two SuggestionContexte implementations for the special suggestion type under test
*/
protected abstract void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion);
private SB mutate(SB firstBuilder) throws IOException {
SB mutation = serializedCopy(firstBuilder);
assertNotSame(mutation, firstBuilder);
@ -201,14 +344,16 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
}
/**
* take and input {@link SuggestBuilder} and return another one that is different in one aspect (to test non-equality)
* take and input {@link SuggestBuilder} and return another one that is
* different in one aspect (to test non-equality)
*/
protected abstract void mutateSpecificParameters(SB firstBuilder) throws IOException;
@SuppressWarnings("unchecked")
protected SB serializedCopy(SB original) throws IOException {
try (BytesStreamOutput output = new BytesStreamOutput()) {
output.writeSuggestion(original);;
output.writeSuggestion(original);
;
try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
return (SB) in.readSuggestion();
}
@ -222,7 +367,8 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
}
/**
* helper to get a random value in a certain range that's different from the input
* helper to get a random value in a certain range that's different from the
* input
*/
protected static <T> T randomValueOtherThan(T input, Supplier<T> randomSupplier) {
T randomValue = null;

9
core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.search.suggest;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.CharsRefBuilder;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.QueryShardContext;
import java.io.IOException;
import java.util.Locale;
@ -54,9 +55,9 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions
@Override
public SuggestContextParser getContextParser() {
return (parser, mapperService, fieldData) -> {
return (parser, shardContext) -> {
Map<String, Object> options = parser.map();
CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(CustomSuggester.this, options);
CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(shardContext, options);
suggestionContext.setField((String) options.get("field"));
return suggestionContext;
};
@ -66,8 +67,8 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions
public Map<String, Object> options;
public CustomSuggestionsContext(Suggester suggester, Map<String, Object> options) {
super(suggester);
public CustomSuggestionsContext(QueryShardContext context, Map<String, Object> options) {
super(new CustomSuggester(), context);
this.options = options;
}
}

8
core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
View File

@ -25,7 +25,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
@ -132,6 +134,12 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
return new CustomSuggestionBuilder(name, randomField, randomSuffix);
}
@Override
protected SuggestionContext build(QueryShardContext context) throws IOException {
// NORELEASE
return null;
}
}
}

10
core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
View File

@ -32,12 +32,12 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.search.suggest.completion.WritableTestCase;
import org.elasticsearch.search.suggest.phrase.Laplace;
import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilderTests;
import org.elasticsearch.search.suggest.phrase.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.StupidBackoff;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
import java.io.IOException;
@ -62,7 +62,7 @@ public class SuggestBuilderTests extends WritableTestCase<SuggestBuilder> {
* creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
*/
public void testFromXContent() throws IOException {
Suggesters suggesters = new Suggesters(Collections.emptyMap(), null, null);
Suggesters suggesters = new Suggesters(Collections.emptyMap());
QueryParseContext context = new QueryParseContext(null);
context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {

52
core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
View File

@ -34,15 +34,10 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperBuilders;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.StringFieldMapper;
import org.elasticsearch.index.mapper.core.StringFieldMapper.StringFieldType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
@ -171,19 +166,10 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
}
};
QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, mockMapperService, null, null, null) {
@Override
public MappedFieldType fieldMapper(String name) {
StringFieldMapper.Builder builder = MapperBuilders.stringField(name);
return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType();
}
};
mockShardContext.setMapUnmappedFieldAsString(true);
for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {
DirectCandidateGeneratorBuilder generator = randomCandidateGenerator();
// first, build via DirectCandidateGenerator#build()
DirectCandidateGenerator contextGenerator = generator.build(mockShardContext);
DirectCandidateGenerator contextGenerator = generator.build(mockMapperService);
// second, render random test generator to xContent and parse using
// PhraseSuggestParser
@ -195,28 +181,32 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
XContentParser parser = XContentHelper.createParser(builder.bytes());
DirectCandidateGenerator secondGenerator = PhraseSuggestParser.parseCandidateGenerator(parser,
mockShardContext.getMapperService(), mockShardContext.parseFieldMatcher());
mockMapperService, ParseFieldMatcher.EMPTY);
// compare their properties
assertNotSame(contextGenerator, secondGenerator);
assertEquals(contextGenerator.field(), secondGenerator.field());
assertEquals(contextGenerator.accuracy(), secondGenerator.accuracy(), Float.MIN_VALUE);
assertEquals(contextGenerator.maxTermFreq(), secondGenerator.maxTermFreq(), Float.MIN_VALUE);
assertEquals(contextGenerator.maxEdits(), secondGenerator.maxEdits());
assertEquals(contextGenerator.maxInspections(), secondGenerator.maxInspections());
assertEquals(contextGenerator.minDocFreq(), secondGenerator.minDocFreq(), Float.MIN_VALUE);
assertEquals(contextGenerator.minWordLength(), secondGenerator.minWordLength());
assertEquals(contextGenerator.postFilter(), secondGenerator.postFilter());
assertEquals(contextGenerator.prefixLength(), secondGenerator.prefixLength());
assertEquals(contextGenerator.preFilter(), secondGenerator.preFilter());
assertEquals(contextGenerator.sort(), secondGenerator.sort());
assertEquals(contextGenerator.size(), secondGenerator.size());
// some instances of StringDistance don't support equals, just checking the class here
assertEquals(contextGenerator.stringDistance().getClass(), secondGenerator.stringDistance().getClass());
assertEquals(contextGenerator.suggestMode(), secondGenerator.suggestMode());
assertEqualGenerators(contextGenerator, secondGenerator);
}
}
public static void assertEqualGenerators(DirectCandidateGenerator first, DirectCandidateGenerator second) {
assertEquals(first.field(), second.field());
assertEquals(first.accuracy(), second.accuracy(), Float.MIN_VALUE);
assertEquals(first.maxTermFreq(), second.maxTermFreq(), Float.MIN_VALUE);
assertEquals(first.maxEdits(), second.maxEdits());
assertEquals(first.maxInspections(), second.maxInspections());
assertEquals(first.minDocFreq(), second.minDocFreq(), Float.MIN_VALUE);
assertEquals(first.minWordLength(), second.minWordLength());
assertEquals(first.postFilter(), second.postFilter());
assertEquals(first.prefixLength(), second.prefixLength());
assertEquals(first.preFilter(), second.preFilter());
assertEquals(first.sort(), second.sort());
assertEquals(first.size(), second.size());
// some instances of StringDistance don't support equals, just checking the class here
assertEquals(first.stringDistance().getClass(), second.stringDistance().getClass());
assertEquals(first.suggestMode(), second.suggestMode());
}
/**
* test that bad xContent throws exception
*/

3
core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java
View File

@ -19,9 +19,6 @@
package org.elasticsearch.search.suggest.phrase;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import static org.hamcrest.Matchers.instanceOf;
public class LaplaceModelTests extends SmoothingModelTestCase {

3
core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java
View File

@ -19,9 +19,6 @@
package org.elasticsearch.search.suggest.phrase;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import static org.hamcrest.Matchers.instanceOf;
public class LinearInterpolationModelTests extends SmoothingModelTestCase {

47
core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
View File

@ -21,16 +21,17 @@ package org.elasticsearch.search.suggest.phrase;
import org.elasticsearch.script.Template;
import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
import org.junit.BeforeClass;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import static org.hamcrest.Matchers.instanceOf;
public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<PhraseSuggestionBuilder> {
@BeforeClass
@ -70,7 +71,7 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
}
maybeSet(testBuilder::gramSize, randomIntBetween(1, 5));
maybeSet(testBuilder::forceUnigrams, randomBoolean());
maybeSet(testBuilder::tokenLimit, randomInt(20));
maybeSet(testBuilder::tokenLimit, randomIntBetween(1, 20));
if (randomBoolean()) {
testBuilder.smoothingModel(randomSmoothingModel());
}
@ -115,7 +116,7 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
builder.gramSize(randomValueOtherThan(builder.gramSize(), () -> randomIntBetween(1, 5)));
break;
case 4:
builder.tokenLimit(randomValueOtherThan(builder.tokenLimit(), () -> randomInt(20)));
builder.tokenLimit(randomValueOtherThan(builder.tokenLimit(), () -> randomIntBetween(1, 20)));
break;
case 5:
builder.separator(randomValueOtherThan(builder.separator(), () -> randomAsciiOfLengthBetween(1, 10)));
@ -158,4 +159,38 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
}
}
@Override
protected void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion) {
assertThat(oldSuggestion, instanceOf(PhraseSuggestionContext.class));
assertThat(newSuggestion, instanceOf(PhraseSuggestionContext.class));
PhraseSuggestionContext oldPhraseSuggestion = (PhraseSuggestionContext) oldSuggestion;
PhraseSuggestionContext newPhraseSuggestion = (PhraseSuggestionContext) newSuggestion;
assertEquals(oldPhraseSuggestion.confidence(), newPhraseSuggestion.confidence(), Float.MIN_VALUE);
assertEquals(oldPhraseSuggestion.collatePrune(), newPhraseSuggestion.collatePrune());
assertEquals(oldPhraseSuggestion.gramSize(), newPhraseSuggestion.gramSize());
assertEquals(oldPhraseSuggestion.realworldErrorLikelyhood(), newPhraseSuggestion.realworldErrorLikelyhood(), Float.MIN_VALUE);
assertEquals(oldPhraseSuggestion.maxErrors(), newPhraseSuggestion.maxErrors(), Float.MIN_VALUE);
assertEquals(oldPhraseSuggestion.separator(), newPhraseSuggestion.separator());
assertEquals(oldPhraseSuggestion.getTokenLimit(), newPhraseSuggestion.getTokenLimit());
assertEquals(oldPhraseSuggestion.getRequireUnigram(), newPhraseSuggestion.getRequireUnigram());
assertEquals(oldPhraseSuggestion.getPreTag(), newPhraseSuggestion.getPreTag());
assertEquals(oldPhraseSuggestion.getPostTag(), newPhraseSuggestion.getPostTag());
if (oldPhraseSuggestion.getCollateQueryScript() != null) {
// only assert that we have a compiled script on the other side
assertNotNull(newPhraseSuggestion.getCollateQueryScript());
}
if (oldPhraseSuggestion.generators() != null) {
assertNotNull(newPhraseSuggestion.generators());
assertEquals(oldPhraseSuggestion.generators().size(), newPhraseSuggestion.generators().size());
Iterator<DirectCandidateGenerator> secondList = newPhraseSuggestion.generators().iterator();
for (DirectCandidateGenerator candidateGenerator : newPhraseSuggestion.generators()) {
DirectCandidateGeneratorTests.assertEqualGenerators(candidateGenerator, secondList.next());
}
}
assertEquals(oldPhraseSuggestion.getCollateScriptParams(), newPhraseSuggestion.getCollateScriptParams());
if (oldPhraseSuggestion.model() != null) {
assertNotNull(newPhraseSuggestion.model());
}
}
}

4
core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
View File

@ -45,10 +45,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
import org.elasticsearch.test.ESTestCase;
import org.junit.AfterClass;
import org.junit.BeforeClass;

3
core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java
View File

@ -19,9 +19,6 @@
package org.elasticsearch.search.suggest.phrase;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
import static org.hamcrest.Matchers.instanceOf;
public class StupidBackoffModelTests extends SmoothingModelTestCase {

14
core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.search.suggest.term;
import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
@ -33,6 +34,14 @@ import static org.hamcrest.Matchers.notNullValue;
*/
public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<TermSuggestionBuilder> {
/**
* creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
*/
@Override
public void testBuild() throws IOException {
// skip for now
}
@Override
protected TermSuggestionBuilder randomSuggestionBuilder() {
TermSuggestionBuilder testBuilder = new TermSuggestionBuilder(randomAsciiOfLength(10));
@ -245,4 +254,9 @@ public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCas
assertThat(builder.suggestMode(), notNullValue());
}
@Override
protected void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion) {
// put assertions on TermSuggestionContext here
}
}

108
modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java
View File

@ -20,38 +20,6 @@
package org.elasticsearch.messy.tests;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion;
import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
import static org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.candidateGenerator;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.nullValue;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
@ -71,13 +39,47 @@ import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder;
import org.elasticsearch.search.suggest.phrase.Laplace;
import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.StupidBackoff;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion;
import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.nullValue;
/**
* Integration tests for term and phrase suggestions. Many of these tests many requests that vary only slightly from one another. Where
* possible these tests should declare for the first request, make the request, modify the configuration for the next request, make that
@ -227,6 +229,16 @@ public class SuggestSearchTests extends ESIntegTestCase {
assertSuggestionSize(searchSuggest, 0, 0, "did_you_mean");
}
/**
* Creates a new {@link DirectCandidateGeneratorBuilder}
*
* @param field
* the field this candidate generator operates on.
*/
private DirectCandidateGeneratorBuilder candidateGenerator(String field) {
return new DirectCandidateGeneratorBuilder(field);
}
// see #2729
public void testSizeOneShard() throws Exception {
prepareCreate("test").setSettings(
@ -286,7 +298,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
refresh();
PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("did_you_mean").field("name.shingled")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
.addCandidateGenerator(candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
.gramSize(3);
Suggest searchSuggest = searchSuggest( "ice tea", phraseSuggestion);
assertSuggestion(searchSuggest, 0, 0, "did_you_mean", "iced tea");
@ -439,7 +451,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
Suggest searchSuggest = searchSuggest( "a an the",
phraseSuggestion("simple_phrase").field("body").gramSize(1)
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always"))
.addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
.size(1));
assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
}
@ -475,13 +487,13 @@ public class SuggestSearchTests extends ESIntegTestCase {
Suggest searchSuggest = searchSuggest( "hello word",
phraseSuggestion("simple_phrase").field("body")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always"))
.addCandidateGenerator(candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always"))
.size(1).confidence(1.0f));
assertSuggestion(searchSuggest, 0, "simple_phrase", "hello words");
searchSuggest = searchSuggest( "hello word",
phraseSuggestion("simple_phrase").field("body")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always"))
.addCandidateGenerator(candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always"))
.size(1).confidence(1.0f));
assertSuggestion(searchSuggest, 0, "simple_phrase", "hello world");
}
@ -573,17 +585,17 @@ public class SuggestSearchTests extends ESIntegTestCase {
// set all mass to trigrams (not indexed)
phraseSuggest.clearCandidateGenerators()
.addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(1,0,0));
.smoothingModel(new LinearInterpolation(1,0,0));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
// set all mass to bigrams
phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0,1,0));
phraseSuggest.smoothingModel(new LinearInterpolation(0,1,0));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
// distribute mass
phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0.4,0.4,0.2));
phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
@ -591,15 +603,15 @@ public class SuggestSearchTests extends ESIntegTestCase {
assertSuggestion(searchSuggest, 0, "simple_phrase", "american ace");
// try all smoothing methods
phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0.4,0.4,0.2));
phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.Laplace(0.2));
phraseSuggest.smoothingModel(new Laplace(0.2));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1));
phraseSuggest.smoothingModel(new StupidBackoff(0.1));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
@ -608,7 +620,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
phraseSuggest.tokenLimit(15).smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1));
phraseSuggest.tokenLimit(15).smoothingModel(new StupidBackoff(0.1));
searchSuggest = searchSuggest( "Xor the Got-Jewel Xor the Got-Jewel Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel xorr the god jewel xorr the god jewel");
// Check the name this time because we're repeating it which is funky
@ -671,7 +683,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
.gramSize(2)
.analyzer("body")
.addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(1).accuracy(0.1f))
.smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1))
.smoothingModel(new StupidBackoff(0.1))
.maxErrors(1.0f)
.size(5);
Suggest searchSuggest = searchSuggest( "Xorr the Gut-Jewel", phraseSuggestion);
@ -931,7 +943,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
Suggest searchSuggest = searchSuggest("nobel prize", phraseSuggestion("simple_phrase")
.field("body")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
.addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
.confidence(2f)
.maxErrors(5f)
.size(1));
@ -939,7 +951,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
searchSuggest = searchSuggest("noble prize", phraseSuggestion("simple_phrase")
.field("body")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
.addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
.confidence(2f)
.maxErrors(5f)
.size(1));
@ -1070,7 +1082,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
PhraseSuggestionBuilder suggest = phraseSuggestion("title")
.field("title")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title")
.addCandidateGenerator(candidateGenerator("title")
.suggestMode("always")
.maxTermFreq(.99f)
.size(1000) // Setting a silly high size helps of generate a larger list of candidates for testing.
@ -1135,7 +1147,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
// suggest without collate
PhraseSuggestionBuilder suggest = phraseSuggestion("title")
.field("title")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title")
.addCandidateGenerator(candidateGenerator("title")
.suggestMode("always")
.maxTermFreq(.99f)
.size(10)