honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-09 14:34:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16507 from cbuescher/phrase-suggest-build

Browse Source 
Add build method to create SuggestionContext to PhraseSuggestionBuilder
This commit is contained in:
Christoph Büscher 2016-02-10 19:05:13 +01:00
commit bbeb09eae7
42 changed files with 1135 additions and 658 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java
View File

@ -142,8 +142,7 @@ public class TransportSuggestAction extends TransportBroadcastAction<SuggestRequ
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new IllegalArgumentException("suggest content missing");
}
final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.mapperService(),
indexService.fieldData(), request.shardId().getIndexName(), request.shardId().id());
final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.newQueryShardContext());
final Suggest result = suggestPhase.execute(context, searcher.searcher());
return new ShardSuggestResponse(request.shardId(), result);
}

2
core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
View File

@ -39,7 +39,7 @@ import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
import org.elasticsearch.search.rescore.RescoreBuilder;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.SmoothingModel;
import org.elasticsearch.tasks.Task;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;

2
core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
View File

@ -38,7 +38,7 @@ import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
import org.elasticsearch.search.rescore.RescoreBuilder;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.SmoothingModel;
import org.elasticsearch.tasks.Task;
import org.joda.time.ReadableInstant;

4
core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java
View File

@ -19,12 +19,12 @@
package org.elasticsearch.indices.query;
import java.util.Map;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryParser;
import java.util.Map;
public class IndicesQueriesRegistry extends AbstractComponent {
private Map<String, QueryParser<?>> queryParsers;

4
core/src/main/java/org/elasticsearch/search/SearchService.java
View File

@ -751,7 +751,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
if (source.rescores() != null) {
try {
for (RescoreBuilder<?> rescore : source.rescores()) {
context.addRescore(rescore.build(context.getQueryShardContext()));
context.addRescore(rescore.build(queryShardContext));
}
} catch (IOException e) {
throw new SearchContextException(context, "failed to create RescoreSearchContext", e);
@ -776,7 +776,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
if (source.highlighter() != null) {
HighlightBuilder highlightBuilder = source.highlighter();
try {
context.highlight(highlightBuilder.build(context.getQueryShardContext()));
context.highlight(highlightBuilder.build(queryShardContext));
} catch (IOException e) {
throw new SearchContextException(context, "failed to create SearchContextHighlighter", e);
}

18
core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java
View File

@ -26,9 +26,12 @@ import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import java.io.IOException;
import java.util.ArrayList;
@ -137,6 +140,21 @@ public class SuggestBuilder extends ToXContentToBytes implements Writeable<Sugge
return suggestBuilder;
}
public SuggestionSearchContext build(QueryShardContext context) throws IOException {
SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext();
for (SuggestionBuilder<?> suggestionBuilder : suggestions) {
SuggestionContext suggestionContext = suggestionBuilder.build(context);
if (suggestionContext.getText() == null) {
if (globalText == null) {
throw new IllegalArgumentException("The required text option is missing");
}
suggestionContext.setText(BytesRefs.toBytesRef(globalText));
}
suggestionSearchContext.addSuggestion(suggestionBuilder.name(), suggestionContext);
}
return suggestionSearchContext;
}
@Override
public SuggestBuilder readFrom(StreamInput in) throws IOException {
final SuggestBuilder builder = new SuggestBuilder();

5
core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java
View File

@ -19,12 +19,11 @@
package org.elasticsearch.search.suggest;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryShardContext;
import java.io.IOException;
public interface SuggestContextParser {
SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService indexFieldDataService) throws IOException;
SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException;
}

26
core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java
View File

@ -21,8 +21,8 @@ package org.elasticsearch.search.suggest;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
@ -44,14 +44,13 @@ public final class SuggestParseElement implements SearchParseElement {
@Override
public void parse(XContentParser parser, SearchContext context) throws Exception {
SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.fieldData(),
context.shardTarget().index(), context.shardTarget().shardId());
SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.getQueryShardContext());
context.suggest(suggestionSearchContext);
}
public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService,
String index, int shardId) throws IOException {
public SuggestionSearchContext parseInternal(XContentParser parser, QueryShardContext shardContext) throws IOException {
SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext();
MapperService mapperService = shardContext.getMapperService();
BytesRef globalText = null;
String fieldName = null;
@ -95,10 +94,20 @@ public final class SuggestParseElement implements SearchParseElement {
throw new IllegalArgumentException("Suggester[" + fieldName + "] not supported");
}
final SuggestContextParser contextParser = suggesters.get(fieldName).getContextParser();
suggestionContext = contextParser.parse(parser, mapperService, fieldDataService);
suggestionContext = contextParser.parse(parser, shardContext);
}
}
if (suggestionContext != null) {
if (suggestText != null) {
suggestionContext.setText(suggestText);
}
if (prefix != null) {
suggestionContext.setPrefix(prefix);
}
if (regex != null) {
suggestionContext.setRegex(regex);
}
if (suggestText != null && prefix == null) {
suggestionContext.setPrefix(suggestText);
suggestionContext.setText(suggestText);
@ -110,6 +119,8 @@ public final class SuggestParseElement implements SearchParseElement {
suggestionContext.setText(regex);
}
suggestionContexts.put(suggestionName, suggestionContext);
} else {
throw new IllegalArgumentException("suggestion context could not be parsed correctly");
}
}
}
@ -117,9 +128,6 @@ public final class SuggestParseElement implements SearchParseElement {
for (Map.Entry<String, SuggestionContext> entry : suggestionContexts.entrySet()) {
String suggestionName = entry.getKey();
SuggestionContext suggestionContext = entry.getValue();
suggestionContext.setShard(shardId);
suggestionContext.setIndex(index);
SuggestUtils.verifySuggestion(mapperService, globalText, suggestionContext);
suggestionSearchContext.addSuggestion(suggestionName, suggestionContext);
}

3
core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java
View File

@ -271,10 +271,10 @@ public final class SuggestUtils {
return false;
}
return true;
}
public static void verifySuggestion(MapperService mapperService, BytesRef globalText, SuggestionContext suggestion) {
// Verify options and set defaults
if (suggestion.getField() == null) {
@ -294,7 +294,6 @@ public final class SuggestUtils {
}
}
public static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) {
if (analyzer instanceof NamedAnalyzer) {
analyzer = ((NamedAnalyzer)analyzer).analyzer();

18
core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java
View File

@ -20,8 +20,6 @@ package org.elasticsearch.search.suggest;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.util.ExtensionPoint;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.suggest.completion.CompletionSuggester;
import org.elasticsearch.search.suggest.phrase.PhraseSuggester;
import org.elasticsearch.search.suggest.term.TermSuggester;
@ -42,21 +40,17 @@ public final class Suggesters extends ExtensionPoint.ClassMap<Suggester> {
this(Collections.emptyMap());
}
@Inject
public Suggesters(Map<String, Suggester> suggesters) {
super("suggester", Suggester.class, new HashSet<>(Arrays.asList("phrase", "term", "completion")), Suggesters.class, SuggestParseElement.class, SuggestPhase.class);
this.parsers = Collections.unmodifiableMap(suggesters);
this.parsers = Collections.unmodifiableMap(addBuildIns(suggesters));
}
@Inject
public Suggesters(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) {
this(addBuildIns(suggesters, scriptService, indexServices));
}
private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) {
private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters) {
final Map<String, Suggester> map = new HashMap<>();
map.put("phrase", new PhraseSuggester(scriptService, indexServices));
map.put("term", new TermSuggester());
map.put("completion", new CompletionSuggester());
map.put("phrase", PhraseSuggester.PROTOTYPE);
map.put("term", TermSuggester.PROTOTYPE);
map.put("completion", CompletionSuggester.PROTOTYPE);
map.putAll(suggesters);
return map;
}

55
core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
View File

@ -19,15 +19,20 @@
package org.elasticsearch.search.suggest;
import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.action.support.ToXContentToBytes;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import java.io.IOException;
import java.util.Objects;
@ -192,6 +197,56 @@ public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends
protected abstract SuggestionBuilder<T> innerFromXContent(QueryParseContext parseContext, String name) throws IOException;
protected abstract SuggestionContext build(QueryShardContext context) throws IOException;
/**
* Transfers the text, prefix, regex, analyzer, fieldname, size and shard size settings from the
* original {@link SuggestionBuilder} to the target {@link SuggestionContext}
*/
protected void populateCommonFields(MapperService mapperService,
SuggestionSearchContext.SuggestionContext suggestionContext) throws IOException {
if (analyzer != null) {
Analyzer luceneAnalyzer = mapperService.analysisService().analyzer(analyzer);
if (luceneAnalyzer == null) {
throw new IllegalArgumentException("Analyzer [" + luceneAnalyzer + "] doesn't exists");
}
suggestionContext.setAnalyzer(luceneAnalyzer);
}
if (fieldname != null) {
suggestionContext.setField(fieldname);
}
if (size != null) {
suggestionContext.setSize(size);
}
if (shardSize != null) {
suggestionContext.setShardSize(shardSize);
} else {
// if no shard size is set in builder, use size (or at least 5)
suggestionContext.setShardSize(Math.max(suggestionContext.getSize(), 5));
}
if (text != null) {
suggestionContext.setText(BytesRefs.toBytesRef(text));
}
if (prefix != null) {
suggestionContext.setPrefix(BytesRefs.toBytesRef(prefix));
}
if (regex != null) {
suggestionContext.setRegex(BytesRefs.toBytesRef(regex));
}
if (text != null && prefix == null) {
suggestionContext.setPrefix(BytesRefs.toBytesRef(text));
} else if (text == null && prefix != null) {
suggestionContext.setText(BytesRefs.toBytesRef(prefix));
} else if (text == null && regex != null) {
suggestionContext.setText(BytesRefs.toBytesRef(regex));
}
}
private String getSuggesterName() {
//default impl returns the same as writeable name, but we keep the distinction between the two just to make sure
return getWriteableName();

41
core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.search.suggest;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.query.QueryShardContext;
import java.util.LinkedHashMap;
import java.util.Map;
@ -36,20 +37,24 @@ public class SuggestionSearchContext {
public Map<String, SuggestionContext> suggestions() {
return suggestions;
}
public static class SuggestionContext {
public abstract static class SuggestionContext {
private BytesRef text;
private BytesRef prefix;
private BytesRef regex;
private final Suggester suggester;
private String field;
private Analyzer analyzer;
private int size = 5;
private int shardSize = -1;
private int shardId;
private String index;
private QueryShardContext shardContext;
private Suggester<?> suggester;
protected SuggestionContext(Suggester<?> suggester, QueryShardContext shardContext) {
this.suggester = suggester;
this.shardContext = shardContext;
}
public BytesRef getText() {
return text;
}
@ -74,12 +79,8 @@ public class SuggestionSearchContext {
this.regex = regex;
}
public SuggestionContext(Suggester suggester) {
this.suggester = suggester;
}
public Suggester<SuggestionContext> getSuggester() {
return this.suggester;
return ((Suggester<SuggestionContext>) suggester);
}
public Analyzer getAnalyzer() {
@ -119,21 +120,9 @@ public class SuggestionSearchContext {
}
this.shardSize = shardSize;
}
public void setShard(int shardId) {
this.shardId = shardId;
}
public void setIndex(String index) {
this.index = index;
}
public String getIndex() {
return index;
}
public int getShard() {
return shardId;
public QueryShardContext getShardContext() {
return this.shardContext;
}
}

8
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java
View File

@ -20,17 +20,16 @@ package org.elasticsearch.search.suggest.completion;
import org.apache.lucene.analysis.Analyzer;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.RegexpFlag;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils.Fields;
@ -135,8 +134,9 @@ public class CompletionSuggestParser implements SuggestContextParser {
}
@Override
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester, mapperService, fieldDataService);
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException {
MapperService mapperService = shardContext.getMapperService();
final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(shardContext);
final ContextAndSuggest contextAndSuggest = new ContextAndSuggest(mapperService);
TLP_PARSER.parse(parser, suggestion, contextAndSuggest);
final XContentParser contextParser = contextAndSuggest.contextParser;

10
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
View File

@ -34,7 +34,9 @@ import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.Suggester;
@ -51,6 +53,8 @@ import java.util.Set;
public class CompletionSuggester extends Suggester<CompletionSuggestionContext> {
public static final CompletionSuggester PROTOTYPE = new CompletionSuggester();
@Override
public SuggestContextParser getContextParser() {
return new CompletionSuggestParser(this);
@ -86,9 +90,11 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
final LeafReaderContext subReaderContext = leaves.get(readerIndex);
final int subDocId = suggestDoc.doc - subReaderContext.docBase;
for (String field : payloadFields) {
MappedFieldType payloadFieldType = suggestionContext.getMapperService().fullName(field);
MapperService mapperService = suggestionContext.getShardContext().getMapperService();
MappedFieldType payloadFieldType = mapperService.fullName(field);
if (payloadFieldType != null) {
final AtomicFieldData data = suggestionContext.getIndexFieldDataService().getForField(payloadFieldType)
QueryShardContext shardContext = suggestionContext.getShardContext();
final AtomicFieldData data = shardContext.getForField(payloadFieldType)
.load(subReaderContext);
final ScriptDocValues scriptValues = data.getScriptValues();
scriptValues.setNextDocId(subDocId);

9
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
View File

@ -28,8 +28,10 @@ import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.RegexpFlag;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
import org.elasticsearch.search.suggest.completion.context.GeoQueryContext;
@ -372,9 +374,16 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
@Override
protected CompletionSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String name) throws IOException {
// NORELEASE
return new CompletionSuggestionBuilder(name);
}
@Override
protected SuggestionContext build(QueryShardContext context) throws IOException {
// NORELEASE
throw new UnsupportedOperationException();
}
@Override
public String getWriteableName() {
return SUGGESTION_NAME;

25
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java
View File

@ -20,10 +20,8 @@ package org.elasticsearch.search.suggest.completion;
import org.apache.lucene.search.suggest.document.CompletionQuery;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.SuggestionSearchContext;
import org.elasticsearch.search.suggest.completion.context.ContextMapping;
import org.elasticsearch.search.suggest.completion.context.ContextMappings;
@ -39,20 +37,16 @@ import java.util.Set;
*/
public class CompletionSuggestionContext extends SuggestionSearchContext.SuggestionContext {
protected CompletionSuggestionContext(QueryShardContext shardContext) {
super(CompletionSuggester.PROTOTYPE, shardContext);
}
private CompletionFieldMapper.CompletionFieldType fieldType;
private CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptionsBuilder;
private CompletionSuggestionBuilder.RegexOptionsBuilder regexOptionsBuilder;
private Map<String, List<ContextMapping.QueryContext>> queryContexts = Collections.emptyMap();
private final MapperService mapperService;
private final IndexFieldDataService indexFieldDataService;
private Set<String> payloadFields = Collections.emptySet();
CompletionSuggestionContext(Suggester suggester, MapperService mapperService, IndexFieldDataService indexFieldDataService) {
super(suggester);
this.indexFieldDataService = indexFieldDataService;
this.mapperService = mapperService;
}
CompletionFieldMapper.CompletionFieldType getFieldType() {
return this.fieldType;
}
@ -73,15 +67,6 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest
this.queryContexts = queryContexts;
}
MapperService getMapperService() {
return mapperService;
}
IndexFieldDataService getIndexFieldDataService() {
return indexFieldDataService;
}
void setPayloadFields(Set<String> fields) {
this.payloadFields = fields;
}

4
core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
View File

@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.CandidateGenerator;
@ -349,8 +348,7 @@ public final class DirectCandidateGeneratorBuilder
return replaceField(tmpFieldName.iterator().next(), tempGenerator);
}
public PhraseSuggestionContext.DirectCandidateGenerator build(QueryShardContext context) throws IOException {
MapperService mapperService = context.getMapperService();
public PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException {
PhraseSuggestionContext.DirectCandidateGenerator generator = new PhraseSuggestionContext.DirectCandidateGenerator();
generator.setField(this.field);
transferIfNotNull(this.size, generator::size);

126
core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java Normal file
View File

@ -0,0 +1,126 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
import java.io.IOException;
import java.util.Objects;
/**
* An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive
* smoothing</a> model.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public final class Laplace extends SmoothingModel {
private double alpha = DEFAULT_LAPLACE_ALPHA;
private static final String NAME = "laplace";
private static final ParseField ALPHA_FIELD = new ParseField("alpha");
static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Default alpha parameter for laplace smoothing
*/
public static final double DEFAULT_LAPLACE_ALPHA = 0.5;
public static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA);
/**
* Creates a Laplace smoothing model.
*
*/
public Laplace(double alpha) {
this.alpha = alpha;
}
/**
* @return the laplace model alpha parameter
*/
public double getAlpha() {
return this.alpha;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(ALPHA_FIELD.getPreferredName(), alpha);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(alpha);
}
@Override
public SmoothingModel readFrom(StreamInput in) throws IOException {
return new Laplace(in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
Laplace otherModel = (Laplace) other;
return Objects.equals(alpha, otherModel.alpha);
}
@Override
protected final int doHashCode() {
return Objects.hash(alpha);
}
@Override
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double alpha = DEFAULT_LAPLACE_ALPHA;
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
}
if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) {
alpha = parser.doubleValue();
}
}
return new Laplace(alpha);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-> new LaplaceScorer(reader, terms, field, realWordLikelyhood, separator, alpha);
}
}

176
core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java Normal file
View File

@ -0,0 +1,176 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
import java.io.IOException;
import java.util.Objects;
/**
* Linear interpolation smoothing model.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public final class LinearInterpolation extends SmoothingModel {
private static final String NAME = "linear";
public static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1);
private final double trigramLambda;
private final double bigramLambda;
private final double unigramLambda;
static final ParseField PARSE_FIELD = new ParseField(NAME);
private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda");
private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda");
private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda");
/**
* Creates a linear interpolation smoothing model.
*
* Note: the lambdas must sum up to one.
*
* @param trigramLambda
* the trigram lambda
* @param bigramLambda
* the bigram lambda
* @param unigramLambda
* the unigram lambda
*/
public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) {
double sum = trigramLambda + bigramLambda + unigramLambda;
if (Math.abs(sum - 1.0) > 0.001) {
throw new IllegalArgumentException("linear smoothing lambdas must sum to 1");
}
this.trigramLambda = trigramLambda;
this.bigramLambda = bigramLambda;
this.unigramLambda = unigramLambda;
}
public double getTrigramLambda() {
return this.trigramLambda;
}
public double getBigramLambda() {
return this.bigramLambda;
}
public double getUnigramLambda() {
return this.unigramLambda;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda);
builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda);
builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(trigramLambda);
out.writeDouble(bigramLambda);
out.writeDouble(unigramLambda);
}
@Override
public LinearInterpolation readFrom(StreamInput in) throws IOException {
return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
final LinearInterpolation otherModel = (LinearInterpolation) other;
return Objects.equals(trigramLambda, otherModel.trigramLambda) &&
Objects.equals(bigramLambda, otherModel.bigramLambda) &&
Objects.equals(unigramLambda, otherModel.unigramLambda);
}
@Override
protected final int doHashCode() {
return Objects.hash(trigramLambda, bigramLambda, unigramLambda);
}
@Override
public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double trigramLambda = 0.0;
double bigramLambda = 0.0;
double unigramLambda = 0.0;
ParseFieldMatcher matcher = parseContext.parseFieldMatcher();
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
if (matcher.match(fieldName, TRIGRAM_FIELD)) {
trigramLambda = parser.doubleValue();
if (trigramLambda < 0) {
throw new IllegalArgumentException("trigram_lambda must be positive");
}
} else if (matcher.match(fieldName, BIGRAM_FIELD)) {
bigramLambda = parser.doubleValue();
if (bigramLambda < 0) {
throw new IllegalArgumentException("bigram_lambda must be positive");
}
} else if (matcher.match(fieldName, UNIGRAM_FIELD)) {
unigramLambda = parser.doubleValue();
if (unigramLambda < 0) {
throw new IllegalArgumentException("unigram_lambda must be positive");
}
} else {
throw new IllegalArgumentException(
"suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(),
"[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) ->
new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda,
unigramLambda);
}
}

16
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java
View File

@ -26,17 +26,16 @@ import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.Template;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.SuggestionSearchContext;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
import java.io.IOException;
@ -51,8 +50,10 @@ public final class PhraseSuggestParser implements SuggestContextParser {
}
@Override
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
PhraseSuggestionContext suggestion = new PhraseSuggestionContext(suggester);
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException {
MapperService mapperService = shardContext.getMapperService();
ScriptService scriptService = shardContext.getScriptService();
PhraseSuggestionContext suggestion = new PhraseSuggestionContext(shardContext);
ParseFieldMatcher parseFieldMatcher = mapperService.getIndexSettings().getParseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
@ -135,7 +136,7 @@ public final class PhraseSuggestParser implements SuggestContextParser {
throw new IllegalArgumentException("suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]");
}
Template template = Template.parse(parser, parseFieldMatcher);
CompiledScript compiledScript = suggester.scriptService().compile(template, ScriptContext.Standard.SEARCH, Collections.emptyMap());
CompiledScript compiledScript = scriptService.compile(template, ScriptContext.Standard.SEARCH, Collections.emptyMap());
suggestion.setCollateQueryScript(compiledScript);
} else if ("params".equals(fieldName)) {
suggestion.setCollateScriptParams(parser.map());
@ -199,9 +200,6 @@ public final class PhraseSuggestParser implements SuggestContextParser {
suggestion.addGenerator(generator);
}
}
return suggestion;
}

20
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
View File

@ -31,9 +31,7 @@ import org.apache.lucene.util.CharsRefBuilder;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.query.ParsedQuery;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.ScriptService;
@ -44,6 +42,7 @@ import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result;
import java.io.IOException;
@ -54,13 +53,8 @@ import java.util.Map;
public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
private final BytesRef SEPARATOR = new BytesRef(" ");
private static final String SUGGESTION_TEMPLATE_VAR_NAME = "suggestion";
private final ScriptService scriptService;
private final IndicesService indicesService;
public PhraseSuggester(ScriptService scriptService, IndicesService indicesService) {
this.scriptService = scriptService;
this.indicesService = indicesService;
}
public static final PhraseSuggester PROTOTYPE = new PhraseSuggester();
/*
* More Ideas:
@ -118,10 +112,10 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
// from the index for a correction, collateMatch is updated
final Map<String, Object> vars = suggestion.getCollateScriptParams();
vars.put(SUGGESTION_TEMPLATE_VAR_NAME, spare.toString());
ScriptService scriptService = suggestion.getShardContext().getScriptService();
final ExecutableScript executable = scriptService.executable(collateScript, vars);
final BytesReference querySource = (BytesReference) executable.run();
IndexService indexService = indicesService.indexService(suggestion.getIndex());
final ParsedQuery parsedQuery = indexService.newQueryShardContext().parse(querySource);
final ParsedQuery parsedQuery = suggestion.getShardContext().parse(querySource);
collateMatch = Lucene.exists(searcher, parsedQuery.query());
}
if (!collateMatch && !collatePrune) {
@ -145,15 +139,11 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
return response;
}
private PhraseSuggestion.Entry buildResultEntry(PhraseSuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) {
private PhraseSuggestion.Entry buildResultEntry(SuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) {
spare.copyUTF8Bytes(suggestion.getText());
return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore);
}
ScriptService scriptService() {
return scriptService;
}
@Override
public SuggestContextParser getContextParser() {
return new PhraseSuggestParser(this);

524
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
View File

@ -19,27 +19,32 @@
package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.Template;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -239,7 +244,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
/**
* Sets an explicit smoothing model used for this suggester. The default is
* {@link PhraseSuggestionBuilder.StupidBackoff}.
* {@link StupidBackoff}.
*/
public PhraseSuggestionBuilder smoothingModel(SmoothingModel model) {
this.model = model;
@ -254,6 +259,9 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
public PhraseSuggestionBuilder tokenLimit(int tokenLimit) {
if (tokenLimit <= 0) {
throw new IllegalArgumentException("token_limit must be >= 1");
}
this.tokenLimit = tokenLimit;
return this;
}
@ -389,413 +397,6 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
return builder;
}
/**
* Creates a new {@link DirectCandidateGeneratorBuilder}
*
* @param field
* the field this candidate generator operates on.
*/
public static DirectCandidateGeneratorBuilder candidateGenerator(String field) {
return new DirectCandidateGeneratorBuilder(field);
}
/**
* A "stupid-backoff" smoothing model simialr to <a
* href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's
* Backoff</a>. This model is used as the default if no model is configured.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public static final class StupidBackoff extends SmoothingModel {
/**
* Default discount parameter for {@link StupidBackoff} smoothing
*/
public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4;
public static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT);
private double discount = DEFAULT_BACKOFF_DISCOUNT;
private static final String NAME = "stupid_backoff";
private static final ParseField DISCOUNT_FIELD = new ParseField("discount");
private static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Creates a Stupid-Backoff smoothing model.
*
* @param discount
* the discount given to lower order ngrams if the higher order ngram doesn't exits
*/
public StupidBackoff(double discount) {
this.discount = discount;
}
/**
* @return the discount parameter of the model
*/
public double getDiscount() {
return this.discount;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(DISCOUNT_FIELD.getPreferredName(), discount);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(discount);
}
@Override
public StupidBackoff readFrom(StreamInput in) throws IOException {
return new StupidBackoff(in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
StupidBackoff otherModel = (StupidBackoff) other;
return Objects.equals(discount, otherModel.discount);
}
@Override
protected final int doHashCode() {
return Objects.hash(discount);
}
@Override
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double discount = DEFAULT_BACKOFF_DISCOUNT;
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
}
if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) {
discount = parser.doubleValue();
}
}
return new StupidBackoff(discount);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount);
}
}
/**
* An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive
* smoothing</a> model.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public static final class Laplace extends SmoothingModel {
private double alpha = DEFAULT_LAPLACE_ALPHA;
private static final String NAME = "laplace";
private static final ParseField ALPHA_FIELD = new ParseField("alpha");
private static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Default alpha parameter for laplace smoothing
*/
public static final double DEFAULT_LAPLACE_ALPHA = 0.5;
public static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA);
/**
* Creates a Laplace smoothing model.
*
*/
public Laplace(double alpha) {
this.alpha = alpha;
}
/**
* @return the laplace model alpha parameter
*/
public double getAlpha() {
return this.alpha;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(ALPHA_FIELD.getPreferredName(), alpha);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(alpha);
}
@Override
public SmoothingModel readFrom(StreamInput in) throws IOException {
return new Laplace(in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
Laplace otherModel = (Laplace) other;
return Objects.equals(alpha, otherModel.alpha);
}
@Override
protected final int doHashCode() {
return Objects.hash(alpha);
}
@Override
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double alpha = DEFAULT_LAPLACE_ALPHA;
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
}
if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) {
alpha = parser.doubleValue();
}
}
return new Laplace(alpha);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-> new LaplaceScorer(reader, terms, field, realWordLikelyhood, separator, alpha);
}
}
public static abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(getWriteableName());
innerToXContent(builder,params);
builder.endObject();
return builder;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
SmoothingModel other = (SmoothingModel) obj;
return doEquals(other);
}
public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
SmoothingModel model = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) {
model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) {
model = Laplace.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) {
model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext);
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(),
"[smoothing] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return model;
}
public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException;
@Override
public final int hashCode() {
/*
* Override hashCode here and forward to an abstract method to force extensions of this class to override hashCode in the same
* way that we force them to override equals. This also prevents false positives in CheckStyle's EqualsHashCode check.
*/
return doHashCode();
}
public abstract WordScorerFactory buildWordScorerFactory();
/**
* subtype specific implementation of "equals".
*/
protected abstract boolean doEquals(SmoothingModel other);
protected abstract int doHashCode();
protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
}
/**
* Linear interpolation smoothing model.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public static final class LinearInterpolation extends SmoothingModel {
private static final String NAME = "linear";
public static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1);
private final double trigramLambda;
private final double bigramLambda;
private final double unigramLambda;
private static final ParseField PARSE_FIELD = new ParseField(NAME);
private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda");
private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda");
private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda");
/**
* Creates a linear interpolation smoothing model.
*
* Note: the lambdas must sum up to one.
*
* @param trigramLambda
* the trigram lambda
* @param bigramLambda
* the bigram lambda
* @param unigramLambda
* the unigram lambda
*/
public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) {
double sum = trigramLambda + bigramLambda + unigramLambda;
if (Math.abs(sum - 1.0) > 0.001) {
throw new IllegalArgumentException("linear smoothing lambdas must sum to 1");
}
this.trigramLambda = trigramLambda;
this.bigramLambda = bigramLambda;
this.unigramLambda = unigramLambda;
}
public double getTrigramLambda() {
return this.trigramLambda;
}
public double getBigramLambda() {
return this.bigramLambda;
}
public double getUnigramLambda() {
return this.unigramLambda;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda);
builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda);
builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(trigramLambda);
out.writeDouble(bigramLambda);
out.writeDouble(unigramLambda);
}
@Override
public LinearInterpolation readFrom(StreamInput in) throws IOException {
return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
final LinearInterpolation otherModel = (LinearInterpolation) other;
return Objects.equals(trigramLambda, otherModel.trigramLambda) &&
Objects.equals(bigramLambda, otherModel.bigramLambda) &&
Objects.equals(unigramLambda, otherModel.unigramLambda);
}
@Override
protected final int doHashCode() {
return Objects.hash(trigramLambda, bigramLambda, unigramLambda);
}
@Override
public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double trigramLambda = 0.0;
double bigramLambda = 0.0;
double unigramLambda = 0.0;
ParseFieldMatcher matcher = parseContext.parseFieldMatcher();
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
if (matcher.match(fieldName, TRIGRAM_FIELD)) {
trigramLambda = parser.doubleValue();
if (trigramLambda < 0) {
throw new IllegalArgumentException("trigram_lambda must be positive");
}
} else if (matcher.match(fieldName, BIGRAM_FIELD)) {
bigramLambda = parser.doubleValue();
if (bigramLambda < 0) {
throw new IllegalArgumentException("bigram_lambda must be positive");
}
} else if (matcher.match(fieldName, UNIGRAM_FIELD)) {
unigramLambda = parser.doubleValue();
if (unigramLambda < 0) {
throw new IllegalArgumentException("unigram_lambda must be positive");
}
} else {
throw new IllegalArgumentException(
"suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(),
"[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) ->
new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda,
unigramLambda);
}
}
@Override
protected PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String suggestionName) throws IOException {
XContentParser parser = parseContext.parser();
@ -873,7 +474,6 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
"suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]");
}
Template template = Template.parse(parser, parseFieldMatcher);
// TODO remember to compile script in build() method
suggestion.collateQuery(template);
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS)) {
suggestion.collateParams(parser.map());
@ -898,6 +498,98 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
return suggestion;
}
@Override
public SuggestionContext build(QueryShardContext context) throws IOException {
PhraseSuggestionContext suggestionContext = new PhraseSuggestionContext(context);
MapperService mapperService = context.getMapperService();
populateCommonFields(mapperService, suggestionContext);
suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator));
suggestionContext.setRealWordErrorLikelihood(this.realWordErrorLikelihood);
suggestionContext.setConfidence(this.confidence);
suggestionContext.setMaxErrors(this.maxErrors);
suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator));
suggestionContext.setRequireUnigram(this.forceUnigrams);
suggestionContext.setTokenLimit(this.tokenLimit);
suggestionContext.setPreTag(BytesRefs.toBytesRef(this.preTag));
suggestionContext.setPostTag(BytesRefs.toBytesRef(this.postTag));
if (this.gramSize != null) {
suggestionContext.setGramSize(this.gramSize);
}
for (List<CandidateGenerator> candidateGenerators : this.generators.values()) {
for (CandidateGenerator candidateGenerator : candidateGenerators) {
suggestionContext.addGenerator(candidateGenerator.build(mapperService));
}
}
if (this.model != null) {
suggestionContext.setModel(this.model.buildWordScorerFactory());
}
if (this.collateQuery != null) {
CompiledScript compiledScript = context.getScriptService().compile(this.collateQuery, ScriptContext.Standard.SEARCH,
Collections.emptyMap());
suggestionContext.setCollateQueryScript(compiledScript);
if (this.collateParams != null) {
suggestionContext.setCollateScriptParams(this.collateParams);
}
suggestionContext.setCollatePrune(this.collatePrune);
}
// TODO remove this when field is mandatory in builder ctor
if (suggestionContext.getField() == null) {
throw new IllegalArgumentException("The required field option is missing");
}
MappedFieldType fieldType = mapperService.fullName(suggestionContext.getField());
if (fieldType == null) {
throw new IllegalArgumentException("No mapping found for field [" + suggestionContext.getField() + "]");
} else if (suggestionContext.getAnalyzer() == null) {
// no analyzer name passed in, so try the field's analyzer, or the default analyzer
if (fieldType.searchAnalyzer() == null) {
suggestionContext.setAnalyzer(mapperService.searchAnalyzer());
} else {
suggestionContext.setAnalyzer(fieldType.searchAnalyzer());
}
}
if (suggestionContext.model() == null) {
suggestionContext.setModel(StupidBackoffScorer.FACTORY);
}
if (this.gramSize == null || suggestionContext.generators().isEmpty()) {
final ShingleTokenFilterFactory.Factory shingleFilterFactory = SuggestUtils
.getShingleFilterFactory(suggestionContext.getAnalyzer());
if (this.gramSize == null) {
// try to detect the shingle size
if (shingleFilterFactory != null) {
suggestionContext.setGramSize(shingleFilterFactory.getMaxShingleSize());
if (suggestionContext.getAnalyzer() == null && shingleFilterFactory.getMinShingleSize() > 1
&& !shingleFilterFactory.getOutputUnigrams()) {
throw new IllegalArgumentException("The default analyzer for field: [" + suggestionContext.getField()
+ "] doesn't emit unigrams. If this is intentional try to set the analyzer explicitly");
}
}
}
if (suggestionContext.generators().isEmpty()) {
if (shingleFilterFactory != null && shingleFilterFactory.getMinShingleSize() > 1
&& !shingleFilterFactory.getOutputUnigrams() && suggestionContext.getRequireUnigram()) {
throw new IllegalArgumentException("The default candidate generator for phrase suggest can't operate on field: ["
+ suggestionContext.getField() + "] since it doesn't emit unigrams. "
+ "If this is intentional try to set the candidate generator field explicitly");
}
// use a default generator on the same field
DirectCandidateGenerator generator = new DirectCandidateGenerator();
generator.setField(suggestionContext.getField());
suggestionContext.addGenerator(generator);
}
}
return suggestionContext;
}
private static void ensureNoSmoothing(PhraseSuggestionBuilder suggestion) {
if (suggestion.smoothingModel() != null) {
throw new IllegalArgumentException("only one smoothing model supported");
@ -1010,5 +702,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
String getType();
CandidateGenerator fromXContent(QueryParseContext parseContext) throws IOException;
PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException;
}
}

9
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
View File

@ -20,9 +20,9 @@ package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import java.util.ArrayList;
@ -54,8 +54,8 @@ class PhraseSuggestionContext extends SuggestionContext {
private Map<String, Object> collateScriptParams = new HashMap<>(1);
private WordScorer.WordScorerFactory scorer;
public PhraseSuggestionContext(Suggester<? extends PhraseSuggestionContext> suggester) {
super(suggester);
public PhraseSuggestionContext(QueryShardContext shardContext) {
super(PhraseSuggester.PROTOTYPE, shardContext);
}
public float maxErrors() {
@ -154,8 +154,6 @@ class PhraseSuggestionContext extends SuggestionContext {
public void postFilter(Analyzer postFilter) {
this.postFilter = postFilter;
}
}
public void setRequireUnigram(boolean requireUnigram) {
@ -213,5 +211,4 @@ class PhraseSuggestionContext extends SuggestionContext {
boolean collatePrune() {
return prune;
}
}

105
core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java Normal file
View File

@ -0,0 +1,105 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.suggest.phrase;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
import java.io.IOException;
public abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(getWriteableName());
innerToXContent(builder,params);
builder.endObject();
return builder;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
SmoothingModel other = (SmoothingModel) obj;
return doEquals(other);
}
@Override
public final int hashCode() {
/*
* Override hashCode here and forward to an abstract method to force
* extensions of this class to override hashCode in the same way that we
* force them to override equals. This also prevents false positives in
* CheckStyle's EqualsHashCode check.
*/
return doHashCode();
}
protected abstract int doHashCode();
public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
SmoothingModel model = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) {
model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) {
model = Laplace.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) {
model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext);
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(),
"[smoothing] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return model;
}
public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException;
public abstract WordScorerFactory buildWordScorerFactory();
/**
* subtype specific implementation of "equals".
*/
protected abstract boolean doEquals(SmoothingModel other);
protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
}

129
core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java Normal file
View File

@ -0,0 +1,129 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
import java.io.IOException;
import java.util.Objects;
/**
* A "stupid-backoff" smoothing model simialr to <a
* href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's
* Backoff</a>. This model is used as the default if no model is configured.
* <p>
* See <a
* href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
* Smoothing</a> for details.
* </p>
*/
public final class StupidBackoff extends SmoothingModel {
/**
* Default discount parameter for {@link StupidBackoff} smoothing
*/
public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4;
public static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT);
private double discount = DEFAULT_BACKOFF_DISCOUNT;
private static final String NAME = "stupid_backoff";
private static final ParseField DISCOUNT_FIELD = new ParseField("discount");
static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Creates a Stupid-Backoff smoothing model.
*
* @param discount
* the discount given to lower order ngrams if the higher order ngram doesn't exits
*/
public StupidBackoff(double discount) {
this.discount = discount;
}
/**
* @return the discount parameter of the model
*/
public double getDiscount() {
return this.discount;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(DISCOUNT_FIELD.getPreferredName(), discount);
return builder;
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeDouble(discount);
}
@Override
public StupidBackoff readFrom(StreamInput in) throws IOException {
return new StupidBackoff(in.readDouble());
}
@Override
protected boolean doEquals(SmoothingModel other) {
StupidBackoff otherModel = (StupidBackoff) other;
return Objects.equals(discount, otherModel.discount);
}
@Override
protected final int doHashCode() {
return Objects.hash(discount);
}
@Override
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
double discount = DEFAULT_BACKOFF_DISCOUNT;
while ((token = parser.nextToken()) != Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
}
if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) {
discount = parser.doubleValue();
}
}
return new StupidBackoff(discount);
}
@Override
public WordScorerFactory buildWordScorerFactory() {
return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount);
}
}

7
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java
View File

@ -20,8 +20,8 @@ package org.elasticsearch.search.suggest.term;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils;
@ -38,10 +38,11 @@ public final class TermSuggestParser implements SuggestContextParser {
}
@Override
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException {
MapperService mapperService = shardContext.getMapperService();
XContentParser.Token token;
String fieldName = null;
TermSuggestionContext suggestion = new TermSuggestionContext(suggester);
TermSuggestionContext suggestion = new TermSuggestionContext(shardContext);
DirectSpellcheckerSettings settings = suggestion.getDirectSpellCheckerSettings();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {

2
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
View File

@ -40,6 +40,8 @@ import java.util.List;
public final class TermSuggester extends Suggester<TermSuggestionContext> {
public static final TermSuggester PROTOTYPE = new TermSuggester();
@Override
public TermSuggestion innerExecute(String name, TermSuggestionContext suggestion, IndexSearcher searcher, CharsRefBuilder spare)
throws IOException {

8
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
View File

@ -26,7 +26,9 @@ import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import java.io.IOException;
import java.util.Locale;
@ -381,6 +383,12 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
return suggestion;
}
@Override
protected SuggestionContext build(QueryShardContext context) throws IOException {
// NORELEASE
throw new UnsupportedOperationException();
}
@Override
public String getWriteableName() {
return SUGGESTION_NAME;

7
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java
View File

@ -18,20 +18,19 @@
*/
package org.elasticsearch.search.suggest.term;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
final class TermSuggestionContext extends SuggestionContext {
private final DirectSpellcheckerSettings settings = new DirectSpellcheckerSettings();
public TermSuggestionContext(Suggester<? extends TermSuggestionContext> suggester) {
super(suggester);
public TermSuggestionContext(QueryShardContext shardContext) {
super(TermSuggester.PROTOTYPE, shardContext);
}
public DirectSpellcheckerSettings getDirectSpellCheckerSettings() {
return settings;
}
}

2
core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
View File

@ -63,8 +63,8 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.script.ScriptContextRegistry;
import org.elasticsearch.script.ScriptEngineRegistry;
import org.elasticsearch.script.ScriptEngineService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import org.elasticsearch.test.engine.MockEngineFactory;

164
core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.suggest;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
@ -31,19 +32,48 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperBuilders;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.StringFieldMapper;
import org.elasticsearch.index.mapper.core.StringFieldMapper.StringFieldType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptContextRegistry;
import org.elasticsearch.script.ScriptEngineRegistry;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptServiceTests.TestEngineService;
import org.elasticsearch.script.ScriptSettings;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import org.elasticsearch.watcher.ResourceWatcherService;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.function.Consumer;
import java.util.function.Supplier;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
@ -51,13 +81,35 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
private static final int NUMBER_OF_TESTBUILDERS = 20;
protected static NamedWriteableRegistry namedWriteableRegistry;
private static final Suggesters suggesters = new Suggesters(Collections.emptyMap(), null, null);
private static Suggesters suggesters;
private static ScriptService scriptService;
private static SuggestParseElement parseElement;
/**
* setup for the whole base test class
*/
@BeforeClass
public static void init() {
public static void init() throws IOException {
Path genericConfigFolder = createTempDir();
Settings baseSettings = settingsBuilder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put(Environment.PATH_CONF_SETTING.getKey(), genericConfigFolder)
.build();
Environment environment = new Environment(baseSettings);
ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry
.ScriptEngineRegistration(TestEngineService.class, TestEngineService.TYPES)));
ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
scriptService = new ScriptService(baseSettings, environment, Collections.singleton(new TestEngineService()),
new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
@Override
public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params) {
return new CompiledScript(ScriptType.INLINE, "mockName", "mocklang", script);
}
};
suggesters = new Suggesters(Collections.emptyMap());
parseElement = new SuggestParseElement(suggesters);
namedWriteableRegistry = new NamedWriteableRegistry();
namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, TermSuggestionBuilder.PROTOTYPE);
namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, PhraseSuggestionBuilder.PROTOTYPE);
@ -69,7 +121,6 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
namedWriteableRegistry = null;
}
/**
* Test serialization and deserialization of the suggestion builder
*/
@ -88,13 +139,13 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
*/
protected SB randomTestBuilder() {
SB randomSuggestion = randomSuggestionBuilder();
randomSuggestion.field(randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::text, randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::prefix, randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::regex, randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::field, randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::analyzer, randomAsciiOfLengthBetween(2, 20));
maybeSet(randomSuggestion::size, randomIntBetween(1, 20));
maybeSet(randomSuggestion::shardSize, randomInt(20));
maybeSet(randomSuggestion::shardSize, randomIntBetween(1, 20));
return randomSuggestion;
}
@ -137,7 +188,8 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
}
/**
* creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
* creates random suggestion builder, renders it to xContent and back to new
* instance that should be equal to original
*/
public void testFromXContent() throws IOException {
QueryParseContext context = new QueryParseContext(null);
@ -166,6 +218,97 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
}
}
/**
* parses random suggestion builder via old parseElement method and via
* build, comparing the results for equality
*/
public void testBuild() throws IOException {
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), Settings.EMPTY);
AnalysisService mockAnalysisService = new AnalysisService(idxSettings, Collections.emptyMap(), Collections.emptyMap(),
Collections.emptyMap(), Collections.emptyMap()) {
@Override
public NamedAnalyzer analyzer(String name) {
return new NamedAnalyzer(name, new WhitespaceAnalyzer());
}
};
MapperService mockMapperService = new MapperService(idxSettings, mockAnalysisService, null, new IndicesModule().getMapperRegistry(),
null) {
@Override
public MappedFieldType fullName(String fullName) {
return new StringFieldType();
}
};
QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, mockMapperService, null, scriptService,
null) {
@Override
public MappedFieldType fieldMapper(String name) {
StringFieldMapper.Builder builder = MapperBuilders.stringField(name);
return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType();
}
};
mockShardContext.setMapUnmappedFieldAsString(true);
for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
SuggestBuilder suggestBuilder = new SuggestBuilder();
SB suggestionBuilder = randomTestBuilder();
suggestBuilder.addSuggestion(suggestionBuilder);
if (suggestionBuilder.text() == null) {
// we either need suggestion text or global text
suggestBuilder.setText(randomAsciiOfLengthBetween(5, 50));
}
if (suggestionBuilder.text() != null && suggestionBuilder.prefix() != null) {
suggestionBuilder.prefix(null);
}
XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
if (randomBoolean()) {
xContentBuilder.prettyPrint();
}
suggestBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes());
parser.nextToken(); // set cursor to START_OBJECT
SuggestionSearchContext parsedSuggestionSearchContext = parseElement.parseInternal(parser, mockShardContext);
SuggestionSearchContext buildSuggestSearchContext = suggestBuilder.build(mockShardContext);
assertEquals(parsedSuggestionSearchContext.suggestions().size(), buildSuggestSearchContext.suggestions().size());
Iterator<Entry<String, SuggestionContext>> iterator = buildSuggestSearchContext.suggestions().entrySet().iterator();
for (Entry<String, SuggestionContext> entry : parsedSuggestionSearchContext.suggestions().entrySet()) {
Entry<String, SuggestionContext> other = iterator.next();
assertEquals(entry.getKey(), other.getKey());
SuggestionContext oldSchoolContext = entry.getValue();
SuggestionContext newSchoolContext = other.getValue();
assertNotSame(oldSchoolContext, newSchoolContext);
// deep comparison of analyzers is difficult here, but we check they are set or not set
if (oldSchoolContext.getAnalyzer() != null) {
assertNotNull(newSchoolContext.getAnalyzer());
} else {
assertNull(newSchoolContext.getAnalyzer());
}
assertEquals(oldSchoolContext.getField(), newSchoolContext.getField());
assertEquals(oldSchoolContext.getPrefix(), newSchoolContext.getPrefix());
assertEquals(oldSchoolContext.getRegex(), newSchoolContext.getRegex());
assertEquals(oldSchoolContext.getShardSize(), newSchoolContext.getShardSize());
assertEquals(oldSchoolContext.getSize(), newSchoolContext.getSize());
assertEquals(oldSchoolContext.getSuggester().getClass(), newSchoolContext.getSuggester().getClass());
assertEquals(oldSchoolContext.getText(), newSchoolContext.getText());
assertEquals(oldSchoolContext.getClass(), newSchoolContext.getClass());
assertSuggestionContext(oldSchoolContext, newSchoolContext);
}
}
}
/**
* compare two SuggestionContexte implementations for the special suggestion type under test
*/
protected abstract void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion);
private SB mutate(SB firstBuilder) throws IOException {
SB mutation = serializedCopy(firstBuilder);
assertNotSame(mutation, firstBuilder);
@ -201,14 +344,16 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
}
/**
* take and input {@link SuggestBuilder} and return another one that is different in one aspect (to test non-equality)
* take and input {@link SuggestBuilder} and return another one that is
* different in one aspect (to test non-equality)
*/
protected abstract void mutateSpecificParameters(SB firstBuilder) throws IOException;
@SuppressWarnings("unchecked")
protected SB serializedCopy(SB original) throws IOException {
try (BytesStreamOutput output = new BytesStreamOutput()) {
output.writeSuggestion(original);;
output.writeSuggestion(original);
;
try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
return (SB) in.readSuggestion();
}
@ -222,7 +367,8 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
}
/**
* helper to get a random value in a certain range that's different from the input
* helper to get a random value in a certain range that's different from the
* input
*/
protected static <T> T randomValueOtherThan(T input, Supplier<T> randomSupplier) {
T randomValue = null;

9
core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.search.suggest;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.CharsRefBuilder;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.QueryShardContext;
import java.io.IOException;
import java.util.Locale;
@ -54,9 +55,9 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions
@Override
public SuggestContextParser getContextParser() {
return (parser, mapperService, fieldData) -> {
return (parser, shardContext) -> {
Map<String, Object> options = parser.map();
CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(CustomSuggester.this, options);
CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(shardContext, options);
suggestionContext.setField((String) options.get("field"));
return suggestionContext;
};
@ -66,8 +67,8 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions
public Map<String, Object> options;
public CustomSuggestionsContext(Suggester suggester, Map<String, Object> options) {
super(suggester);
public CustomSuggestionsContext(QueryShardContext context, Map<String, Object> options) {
super(new CustomSuggester(), context);
this.options = options;
}
}

8
core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
View File

@ -25,7 +25,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
@ -132,6 +134,12 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
return new CustomSuggestionBuilder(name, randomField, randomSuffix);
}
@Override
protected SuggestionContext build(QueryShardContext context) throws IOException {
// NORELEASE
return null;
}
}
}

10
core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
View File

@ -32,12 +32,12 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.search.suggest.completion.WritableTestCase;
import org.elasticsearch.search.suggest.phrase.Laplace;
import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilderTests;
import org.elasticsearch.search.suggest.phrase.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.StupidBackoff;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
import java.io.IOException;
@ -62,7 +62,7 @@ public class SuggestBuilderTests extends WritableTestCase<SuggestBuilder> {
* creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
*/
public void testFromXContent() throws IOException {
Suggesters suggesters = new Suggesters(Collections.emptyMap(), null, null);
Suggesters suggesters = new Suggesters(Collections.emptyMap());
QueryParseContext context = new QueryParseContext(null);
context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {

52
core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
View File

@ -34,15 +34,10 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperBuilders;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.StringFieldMapper;
import org.elasticsearch.index.mapper.core.StringFieldMapper.StringFieldType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
@ -171,19 +166,10 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
}
};
QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, mockMapperService, null, null, null) {
@Override
public MappedFieldType fieldMapper(String name) {
StringFieldMapper.Builder builder = MapperBuilders.stringField(name);
return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType();
}
};
mockShardContext.setMapUnmappedFieldAsString(true);
for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {
DirectCandidateGeneratorBuilder generator = randomCandidateGenerator();
// first, build via DirectCandidateGenerator#build()
DirectCandidateGenerator contextGenerator = generator.build(mockShardContext);
DirectCandidateGenerator contextGenerator = generator.build(mockMapperService);
// second, render random test generator to xContent and parse using
// PhraseSuggestParser
@ -195,28 +181,32 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
XContentParser parser = XContentHelper.createParser(builder.bytes());
DirectCandidateGenerator secondGenerator = PhraseSuggestParser.parseCandidateGenerator(parser,
mockShardContext.getMapperService(), mockShardContext.parseFieldMatcher());
mockMapperService, ParseFieldMatcher.EMPTY);
// compare their properties
assertNotSame(contextGenerator, secondGenerator);
assertEquals(contextGenerator.field(), secondGenerator.field());
assertEquals(contextGenerator.accuracy(), secondGenerator.accuracy(), Float.MIN_VALUE);
assertEquals(contextGenerator.maxTermFreq(), secondGenerator.maxTermFreq(), Float.MIN_VALUE);
assertEquals(contextGenerator.maxEdits(), secondGenerator.maxEdits());
assertEquals(contextGenerator.maxInspections(), secondGenerator.maxInspections());
assertEquals(contextGenerator.minDocFreq(), secondGenerator.minDocFreq(), Float.MIN_VALUE);
assertEquals(contextGenerator.minWordLength(), secondGenerator.minWordLength());
assertEquals(contextGenerator.postFilter(), secondGenerator.postFilter());
assertEquals(contextGenerator.prefixLength(), secondGenerator.prefixLength());
assertEquals(contextGenerator.preFilter(), secondGenerator.preFilter());
assertEquals(contextGenerator.sort(), secondGenerator.sort());
assertEquals(contextGenerator.size(), secondGenerator.size());
// some instances of StringDistance don't support equals, just checking the class here
assertEquals(contextGenerator.stringDistance().getClass(), secondGenerator.stringDistance().getClass());
assertEquals(contextGenerator.suggestMode(), secondGenerator.suggestMode());
assertEqualGenerators(contextGenerator, secondGenerator);
}
}
public static void assertEqualGenerators(DirectCandidateGenerator first, DirectCandidateGenerator second) {
assertEquals(first.field(), second.field());
assertEquals(first.accuracy(), second.accuracy(), Float.MIN_VALUE);
assertEquals(first.maxTermFreq(), second.maxTermFreq(), Float.MIN_VALUE);
assertEquals(first.maxEdits(), second.maxEdits());
assertEquals(first.maxInspections(), second.maxInspections());
assertEquals(first.minDocFreq(), second.minDocFreq(), Float.MIN_VALUE);
assertEquals(first.minWordLength(), second.minWordLength());
assertEquals(first.postFilter(), second.postFilter());
assertEquals(first.prefixLength(), second.prefixLength());
assertEquals(first.preFilter(), second.preFilter());
assertEquals(first.sort(), second.sort());
assertEquals(first.size(), second.size());
// some instances of StringDistance don't support equals, just checking the class here
assertEquals(first.stringDistance().getClass(), second.stringDistance().getClass());
assertEquals(first.suggestMode(), second.suggestMode());
}
/**
* test that bad xContent throws exception
*/

3
core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java
View File

@ -19,9 +19,6 @@
package org.elasticsearch.search.suggest.phrase;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import static org.hamcrest.Matchers.instanceOf;
public class LaplaceModelTests extends SmoothingModelTestCase {

3
core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java
View File

@ -19,9 +19,6 @@
package org.elasticsearch.search.suggest.phrase;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import static org.hamcrest.Matchers.instanceOf;
public class LinearInterpolationModelTests extends SmoothingModelTestCase {

47
core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
View File

@ -21,16 +21,17 @@ package org.elasticsearch.search.suggest.phrase;
import org.elasticsearch.script.Template;
import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
import org.junit.BeforeClass;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import static org.hamcrest.Matchers.instanceOf;
public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<PhraseSuggestionBuilder> {
@BeforeClass
@ -70,7 +71,7 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
}
maybeSet(testBuilder::gramSize, randomIntBetween(1, 5));
maybeSet(testBuilder::forceUnigrams, randomBoolean());
maybeSet(testBuilder::tokenLimit, randomInt(20));
maybeSet(testBuilder::tokenLimit, randomIntBetween(1, 20));
if (randomBoolean()) {
testBuilder.smoothingModel(randomSmoothingModel());
}
@ -115,7 +116,7 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
builder.gramSize(randomValueOtherThan(builder.gramSize(), () -> randomIntBetween(1, 5)));
break;
case 4:
builder.tokenLimit(randomValueOtherThan(builder.tokenLimit(), () -> randomInt(20)));
builder.tokenLimit(randomValueOtherThan(builder.tokenLimit(), () -> randomIntBetween(1, 20)));
break;
case 5:
builder.separator(randomValueOtherThan(builder.separator(), () -> randomAsciiOfLengthBetween(1, 10)));
@ -158,4 +159,38 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
}
}
@Override
protected void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion) {
assertThat(oldSuggestion, instanceOf(PhraseSuggestionContext.class));
assertThat(newSuggestion, instanceOf(PhraseSuggestionContext.class));
PhraseSuggestionContext oldPhraseSuggestion = (PhraseSuggestionContext) oldSuggestion;
PhraseSuggestionContext newPhraseSuggestion = (PhraseSuggestionContext) newSuggestion;
assertEquals(oldPhraseSuggestion.confidence(), newPhraseSuggestion.confidence(), Float.MIN_VALUE);
assertEquals(oldPhraseSuggestion.collatePrune(), newPhraseSuggestion.collatePrune());
assertEquals(oldPhraseSuggestion.gramSize(), newPhraseSuggestion.gramSize());
assertEquals(oldPhraseSuggestion.realworldErrorLikelyhood(), newPhraseSuggestion.realworldErrorLikelyhood(), Float.MIN_VALUE);
assertEquals(oldPhraseSuggestion.maxErrors(), newPhraseSuggestion.maxErrors(), Float.MIN_VALUE);
assertEquals(oldPhraseSuggestion.separator(), newPhraseSuggestion.separator());
assertEquals(oldPhraseSuggestion.getTokenLimit(), newPhraseSuggestion.getTokenLimit());
assertEquals(oldPhraseSuggestion.getRequireUnigram(), newPhraseSuggestion.getRequireUnigram());
assertEquals(oldPhraseSuggestion.getPreTag(), newPhraseSuggestion.getPreTag());
assertEquals(oldPhraseSuggestion.getPostTag(), newPhraseSuggestion.getPostTag());
if (oldPhraseSuggestion.getCollateQueryScript() != null) {
// only assert that we have a compiled script on the other side
assertNotNull(newPhraseSuggestion.getCollateQueryScript());
}
if (oldPhraseSuggestion.generators() != null) {
assertNotNull(newPhraseSuggestion.generators());
assertEquals(oldPhraseSuggestion.generators().size(), newPhraseSuggestion.generators().size());
Iterator<DirectCandidateGenerator> secondList = newPhraseSuggestion.generators().iterator();
for (DirectCandidateGenerator candidateGenerator : newPhraseSuggestion.generators()) {
DirectCandidateGeneratorTests.assertEqualGenerators(candidateGenerator, secondList.next());
}
}
assertEquals(oldPhraseSuggestion.getCollateScriptParams(), newPhraseSuggestion.getCollateScriptParams());
if (oldPhraseSuggestion.model() != null) {
assertNotNull(newPhraseSuggestion.model());
}
}
}

4
core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
View File

@ -45,10 +45,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
import org.elasticsearch.test.ESTestCase;
import org.junit.AfterClass;
import org.junit.BeforeClass;

3
core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java
View File

@ -19,9 +19,6 @@
package org.elasticsearch.search.suggest.phrase;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
import static org.hamcrest.Matchers.instanceOf;
public class StupidBackoffModelTests extends SmoothingModelTestCase {

14
core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.search.suggest.term;
import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
@ -33,6 +34,14 @@ import static org.hamcrest.Matchers.notNullValue;
*/
public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<TermSuggestionBuilder> {
/**
* creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
*/
@Override
public void testBuild() throws IOException {
// skip for now
}
@Override
protected TermSuggestionBuilder randomSuggestionBuilder() {
TermSuggestionBuilder testBuilder = new TermSuggestionBuilder(randomAsciiOfLength(10));
@ -245,4 +254,9 @@ public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCas
assertThat(builder.suggestMode(), notNullValue());
}
@Override
protected void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion) {
// put assertions on TermSuggestionContext here
}
}

108
modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java
View File

@ -20,38 +20,6 @@
package org.elasticsearch.messy.tests;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion;
import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
import static org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.candidateGenerator;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.nullValue;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
@ -71,13 +39,47 @@ import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder;
import org.elasticsearch.search.suggest.phrase.Laplace;
import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.StupidBackoff;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion;
import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.nullValue;
/**
* Integration tests for term and phrase suggestions. Many of these tests many requests that vary only slightly from one another. Where
* possible these tests should declare for the first request, make the request, modify the configuration for the next request, make that
@ -227,6 +229,16 @@ public class SuggestSearchTests extends ESIntegTestCase {
assertSuggestionSize(searchSuggest, 0, 0, "did_you_mean");
}
/**
* Creates a new {@link DirectCandidateGeneratorBuilder}
*
* @param field
* the field this candidate generator operates on.
*/
private DirectCandidateGeneratorBuilder candidateGenerator(String field) {
return new DirectCandidateGeneratorBuilder(field);
}
// see #2729
public void testSizeOneShard() throws Exception {
prepareCreate("test").setSettings(
@ -286,7 +298,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
refresh();
PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("did_you_mean").field("name.shingled")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
.addCandidateGenerator(candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
.gramSize(3);
Suggest searchSuggest = searchSuggest( "ice tea", phraseSuggestion);
assertSuggestion(searchSuggest, 0, 0, "did_you_mean", "iced tea");
@ -439,7 +451,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
Suggest searchSuggest = searchSuggest( "a an the",
phraseSuggestion("simple_phrase").field("body").gramSize(1)
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always"))
.addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
.size(1));
assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
}
@ -475,13 +487,13 @@ public class SuggestSearchTests extends ESIntegTestCase {
Suggest searchSuggest = searchSuggest( "hello word",
phraseSuggestion("simple_phrase").field("body")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always"))
.addCandidateGenerator(candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always"))
.size(1).confidence(1.0f));
assertSuggestion(searchSuggest, 0, "simple_phrase", "hello words");
searchSuggest = searchSuggest( "hello word",
phraseSuggestion("simple_phrase").field("body")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always"))
.addCandidateGenerator(candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always"))
.size(1).confidence(1.0f));
assertSuggestion(searchSuggest, 0, "simple_phrase", "hello world");
}
@ -573,17 +585,17 @@ public class SuggestSearchTests extends ESIntegTestCase {
// set all mass to trigrams (not indexed)
phraseSuggest.clearCandidateGenerators()
.addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(1,0,0));
.smoothingModel(new LinearInterpolation(1,0,0));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
// set all mass to bigrams
phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0,1,0));
phraseSuggest.smoothingModel(new LinearInterpolation(0,1,0));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
// distribute mass
phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0.4,0.4,0.2));
phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
@ -591,15 +603,15 @@ public class SuggestSearchTests extends ESIntegTestCase {
assertSuggestion(searchSuggest, 0, "simple_phrase", "american ace");
// try all smoothing methods
phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0.4,0.4,0.2));
phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.Laplace(0.2));
phraseSuggest.smoothingModel(new Laplace(0.2));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1));
phraseSuggest.smoothingModel(new StupidBackoff(0.1));
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
@ -608,7 +620,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
phraseSuggest.tokenLimit(15).smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1));
phraseSuggest.tokenLimit(15).smoothingModel(new StupidBackoff(0.1));
searchSuggest = searchSuggest( "Xor the Got-Jewel Xor the Got-Jewel Xor the Got-Jewel", phraseSuggest);
assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel xorr the god jewel xorr the god jewel");
// Check the name this time because we're repeating it which is funky
@ -671,7 +683,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
.gramSize(2)
.analyzer("body")
.addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(1).accuracy(0.1f))
.smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1))
.smoothingModel(new StupidBackoff(0.1))
.maxErrors(1.0f)
.size(5);
Suggest searchSuggest = searchSuggest( "Xorr the Gut-Jewel", phraseSuggestion);
@ -931,7 +943,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
Suggest searchSuggest = searchSuggest("nobel prize", phraseSuggestion("simple_phrase")
.field("body")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
.addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
.confidence(2f)
.maxErrors(5f)
.size(1));
@ -939,7 +951,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
searchSuggest = searchSuggest("noble prize", phraseSuggestion("simple_phrase")
.field("body")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
.addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
.confidence(2f)
.maxErrors(5f)
.size(1));
@ -1070,7 +1082,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
PhraseSuggestionBuilder suggest = phraseSuggestion("title")
.field("title")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title")
.addCandidateGenerator(candidateGenerator("title")
.suggestMode("always")
.maxTermFreq(.99f)
.size(1000) // Setting a silly high size helps of generate a larger list of candidates for testing.
@ -1135,7 +1147,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
// suggest without collate
PhraseSuggestionBuilder suggest = phraseSuggestion("title")
.field("title")
.addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title")
.addCandidateGenerator(candidateGenerator("title")
.suggestMode("always")
.maxTermFreq(.99f)
.size(10)