honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16354 from cbuescher/phrase-suggest-fromXContent 

This adds parsing from xContent to PhraseSuggestionBuilder and changes properties of PhraseSuggestionBuilder to using default values form PhraseSuggestionContext where possible.
This commit is contained in:
Christoph Büscher 2016-02-08 12:42:14 +01:00
parent 2dec129038 2ae6420fd6
commit c27408bdf5
19 changed files with 481 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
buildSrc/src/main/resources/checkstyle_suppressions.xml
View File

@ -916,7 +916,6 @@
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]SuggestUtils.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]Suggesters.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]CompletionSuggestParser.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]CompletionSuggester.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]CategoryContextMapping.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]ContextMapping.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]GeoContextMapping.java" checks="LineLength" />
@ -927,12 +926,9 @@
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]LinearInterpoatingScorer.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]NoisyChannelSpellChecker.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]PhraseSuggestParser.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]PhraseSuggester.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]PhraseSuggestionBuilder.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]StupidBackoffScorer.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]WordScorer.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]term[/\\]TermSuggestParser.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]term[/\\]TermSuggester.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]RestoreService.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotInfo.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotShardFailure.java" checks="LineLength" />
@ -1446,7 +1442,6 @@
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]ExistsIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]MultiMatchQueryIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]SearchQueryIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]rescore[/\\]QueryRescoreBuilderTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]DuelScrollIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]SearchScrollIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]SearchScrollWithFailingNodesIT.java" checks="LineLength" />
@ -1462,7 +1457,6 @@
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]GeoContextMappingTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]DirectCandidateGeneratorTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]NoisyChannelSpellCheckerTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]SmoothingModelTestCase.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]similarity[/\\]SimilarityIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]AbstractSnapshotIntegTestCase.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]BlobStoreFormatIT.java" checks="LineLength" />

1
core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java
View File

@ -123,7 +123,6 @@ public final class SuggestParseElement implements SearchParseElement {
SuggestUtils.verifySuggestion(mapperService, globalText, suggestionContext);
suggestionSearchContext.addSuggestion(suggestionName, suggestionContext);
}
return suggestionSearchContext;
}
}

10
core/src/main/java/org/elasticsearch/search/suggest/Suggester.java
View File

@ -29,8 +29,18 @@ public abstract class Suggester<T extends SuggestionSearchContext.SuggestionCont
protected abstract Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
innerExecute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException;
/**
* link the suggester to its corresponding {@link SuggestContextParser}
* TODO: This method should eventually be removed by {@link #getBuilderPrototype()} once
* we don't directly parse from xContent to the SuggestionContext any more
*/
public abstract SuggestContextParser getContextParser();
/**
* link the suggester to its corresponding {@link SuggestionBuilder}
*/
public abstract SuggestionBuilder<?> getBuilderPrototype();
public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
execute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
// #3469 We want to ignore empty shards

12
core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java
View File

@ -64,4 +64,16 @@ public final class Suggesters extends ExtensionPoint.ClassMap<Suggester> {
public Suggester get(String type) {
return parsers.get(type);
}
public SuggestionBuilder<?> getSuggestionPrototype(String suggesterName) {
Suggester<?> suggester = parsers.get(suggesterName);
if (suggester == null) {
throw new IllegalArgumentException("suggester with name [" + suggesterName + "] not supported");
}
SuggestionBuilder<?> suggestParser = suggester.getBuilderPrototype();
if (suggestParser == null) {
throw new IllegalArgumentException("suggester with name [" + suggesterName + "] not supported");
}
return suggestParser;
}
}

55
core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
View File

@ -21,10 +21,13 @@ package org.elasticsearch.search.suggest;
import org.elasticsearch.action.support.ToXContentToBytes;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import java.io.IOException;
import java.util.Objects;
@ -138,12 +141,62 @@ public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends
return builder;
}
protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
public static SuggestionBuilder<?> fromXContent(QueryParseContext parseContext, String suggestionName, Suggesters suggesters)
throws IOException {
XContentParser parser = parseContext.parser();
ParseFieldMatcher parsefieldMatcher = parseContext.parseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
String suggestText = null;
String prefix = null;
String regex = null;
SuggestionBuilder<?> suggestionBuilder = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
if (parsefieldMatcher.match(fieldName, TEXT_FIELD)) {
suggestText = parser.text();
} else if (parsefieldMatcher.match(fieldName, PREFIX_FIELD)) {
prefix = parser.text();
} else if (parsefieldMatcher.match(fieldName, REGEX_FIELD)) {
regex = parser.text();
} else {
throw new IllegalArgumentException("[suggestion] does not support [" + fieldName + "]");
}
} else if (token == XContentParser.Token.START_OBJECT) {
if (suggestionName == null) {
throw new IllegalArgumentException("Suggestion must have name");
}
SuggestionBuilder<?> suggestParser = suggesters.getSuggestionPrototype(fieldName);
if (suggestParser == null) {
throw new IllegalArgumentException("Suggester[" + fieldName + "] not supported");
}
suggestionBuilder = suggestParser.innerFromXContent(parseContext, suggestionName);
}
}
if (suggestText != null) {
suggestionBuilder.text(suggestText);
}
if (prefix != null) {
suggestionBuilder.prefix(prefix);
}
if (regex != null) {
suggestionBuilder.regex(regex);
}
return suggestionBuilder;
}
protected abstract SuggestionBuilder<T> innerFromXContent(QueryParseContext parseContext, String name) throws IOException;
private String getSuggesterName() {
//default impl returns the same as writeable name, but we keep the distinction between the two just to make sure
return getWriteableName();
}
protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
/**
* Sets from what field to fetch the candidate suggestions from. This is an

10
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
View File

@ -38,6 +38,7 @@ import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import java.io.IOException;
import java.util.ArrayList;
@ -50,6 +51,7 @@ import java.util.Set;
public class CompletionSuggester extends Suggester<CompletionSuggestionContext> {
@Override
public SuggestContextParser getContextParser() {
return new CompletionSuggestParser(this);
}
@ -86,7 +88,8 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
for (String field : payloadFields) {
MappedFieldType payloadFieldType = suggestionContext.getMapperService().fullName(field);
if (payloadFieldType != null) {
final AtomicFieldData data = suggestionContext.getIndexFieldDataService().getForField(payloadFieldType).load(subReaderContext);
final AtomicFieldData data = suggestionContext.getIndexFieldDataService().getForField(payloadFieldType)
.load(subReaderContext);
final ScriptDocValues scriptValues = data.getScriptValues();
scriptValues.setNextDocId(subDocId);
payload.put(field, new ArrayList<>(scriptValues.getValues()));
@ -262,4 +265,9 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
}
}
}
@Override
public SuggestionBuilder<?> getBuilderPrototype() {
return CompletionSuggestionBuilder.PROTOTYPE;
}
}

8
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
View File

@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.RegexpFlag;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
@ -50,7 +51,7 @@ import java.util.Set;
public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSuggestionBuilder> {
public static final CompletionSuggestionBuilder PROTOTYPE = new CompletionSuggestionBuilder("_na_"); // name doesn't matter
final static String SUGGESTION_NAME = "completion";
static final String SUGGESTION_NAME = "completion";
static final ParseField PAYLOAD_FIELD = new ParseField("payload");
static final ParseField CONTEXTS_FIELD = new ParseField("contexts", "context");
@ -369,6 +370,11 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
return builder;
}
@Override
protected CompletionSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String name) throws IOException {
return new CompletionSuggestionBuilder(name);
}
@Override
public String getWriteableName() {
return SUGGESTION_NAME;

30
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
View File

@ -44,6 +44,7 @@ import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result;
import java.io.IOException;
@ -65,14 +66,14 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
/*
* More Ideas:
* - add ability to find whitespace problems -> we can build a poor mans decompounder with our index based on a automaton?
* - add ability to build different error models maybe based on a confusion matrix?
* - add ability to build different error models maybe based on a confusion matrix?
* - try to combine a token with its subsequent token to find / detect word splits (optional)
* - for this to work we need some way to defined the position length of a candidate
* - phonetic filters could be interesting here too for candidate selection
*/
@Override
public Suggestion<? extends Entry<? extends Option>> innerExecute(String name, PhraseSuggestionContext suggestion, IndexSearcher searcher,
CharsRefBuilder spare) throws IOException {
public Suggestion<? extends Entry<? extends Option>> innerExecute(String name, PhraseSuggestionContext suggestion,
IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
double realWordErrorLikelihood = suggestion.realworldErrorLikelyhood();
final PhraseSuggestion response = new PhraseSuggestion(name, suggestion.getSize());
final IndexReader indexReader = searcher.getIndexReader();
@ -84,21 +85,23 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
DirectSpellChecker directSpellChecker = SuggestUtils.getDirectSpellChecker(generator);
Terms terms = MultiFields.getTerms(indexReader, generator.field());
if (terms != null) {
gens.add(new DirectCandidateGenerator(directSpellChecker, generator.field(), generator.suggestMode(),
indexReader, realWordErrorLikelihood, generator.size(), generator.preFilter(), generator.postFilter(), terms));
gens.add(new DirectCandidateGenerator(directSpellChecker, generator.field(), generator.suggestMode(),
indexReader, realWordErrorLikelihood, generator.size(), generator.preFilter(), generator.postFilter(), terms));
}
}
final String suggestField = suggestion.getField();
final Terms suggestTerms = MultiFields.getTerms(indexReader, suggestField);
if (gens.size() > 0 && suggestTerms != null) {
final NoisyChannelSpellChecker checker = new NoisyChannelSpellChecker(realWordErrorLikelihood, suggestion.getRequireUnigram(), suggestion.getTokenLimit());
final NoisyChannelSpellChecker checker = new NoisyChannelSpellChecker(realWordErrorLikelihood, suggestion.getRequireUnigram(),
suggestion.getTokenLimit());
final BytesRef separator = suggestion.separator();
WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood, separator);
WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood,
separator);
Result checkerResult;
try (TokenStream stream = checker.tokenStream(suggestion.getAnalyzer(), suggestion.getText(), spare, suggestion.getField())) {
checkerResult = checker.getCorrections(stream, new MultiCandidateGeneratorWrapper(suggestion.getShardSize(),
gens.toArray(new CandidateGenerator[gens.size()])), suggestion.maxErrors(),
suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize());
checkerResult = checker.getCorrections(stream,
new MultiCandidateGeneratorWrapper(suggestion.getShardSize(), gens.toArray(new CandidateGenerator[gens.size()])),
suggestion.maxErrors(), suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize());
}
PhraseSuggestion.Entry resultEntry = buildResultEntry(suggestion, spare, checkerResult.cutoffScore);
@ -152,10 +155,15 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
ScriptService scriptService() {
return scriptService;
}
@Override
public SuggestContextParser getContextParser() {
return new PhraseSuggestParser(this);
}
@Override
public SuggestionBuilder<?> getBuilderPrototype() {
return PhraseSuggestionBuilder.PROTOTYPE;
}
}

275
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.BytesRef;
@ -51,24 +52,42 @@ import java.util.Set;
*/
public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSuggestionBuilder> {
static final String SUGGESTION_NAME = "phrase";
private static final String SUGGESTION_NAME = "phrase";
public static final PhraseSuggestionBuilder PROTOTYPE = new PhraseSuggestionBuilder("_na_");
private Float maxErrors;
private String separator;
private Float realWordErrorLikelihood;
private Float confidence;
private final Map<String, List<CandidateGenerator>> generators = new HashMap<>();
protected static final ParseField MAXERRORS_FIELD = new ParseField("max_errors");
protected static final ParseField RWE_LIKELIHOOD_FIELD = new ParseField("real_word_error_likelihood");
protected static final ParseField SEPARATOR_FIELD = new ParseField("separator");
protected static final ParseField CONFIDENCE_FIELD = new ParseField("confidence");
protected static final ParseField GENERATORS_FIELD = new ParseField("shard_size");
protected static final ParseField GRAMSIZE_FIELD = new ParseField("gram_size");
protected static final ParseField SMOOTHING_MODEL_FIELD = new ParseField("smoothing");
protected static final ParseField FORCE_UNIGRAM_FIELD = new ParseField("force_unigrams");
protected static final ParseField TOKEN_LIMIT_FIELD = new ParseField("token_limit");
protected static final ParseField HIGHLIGHT_FIELD = new ParseField("highlight");
protected static final ParseField PRE_TAG_FIELD = new ParseField("pre_tag");
protected static final ParseField POST_TAG_FIELD = new ParseField("post_tag");
protected static final ParseField COLLATE_FIELD = new ParseField("collate");
protected static final ParseField COLLATE_QUERY_FIELD = new ParseField("query");
protected static final ParseField COLLATE_QUERY_PARAMS = new ParseField("params");
protected static final ParseField COLLATE_QUERY_PRUNE = new ParseField("prune");
private float maxErrors = PhraseSuggestionContext.DEFAULT_MAX_ERRORS;
private String separator = PhraseSuggestionContext.DEFAULT_SEPARATOR;
private float realWordErrorLikelihood = PhraseSuggestionContext.DEFAULT_RWE_ERRORLIKELIHOOD;
private float confidence = PhraseSuggestionContext.DEFAULT_CONFIDENCE;
// gramSize needs to be optional although there is a default, if unset parser try to detect and use shingle size
private Integer gramSize;
private SmoothingModel model;
private Boolean forceUnigrams;
private Integer tokenLimit;
private boolean forceUnigrams = PhraseSuggestionContext.DEFAULT_REQUIRE_UNIGRAM;
private int tokenLimit = NoisyChannelSpellChecker.DEFAULT_TOKEN_LIMIT;
private String preTag;
private String postTag;
private Template collateQuery;
private Map<String, Object> collateParams;
private Boolean collatePrune;
private boolean collatePrune = PhraseSuggestionContext.DEFAULT_COLLATE_PRUNE;
private SmoothingModel model;
private final Map<String, List<CandidateGenerator>> generators = new HashMap<>();
public PhraseSuggestionBuilder(String name) {
super(name);
@ -103,7 +122,10 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
* The default is set to <tt>1.0</tt> which corresponds to that only
* corrections with at most 1 missspelled term are returned.
*/
public PhraseSuggestionBuilder maxErrors(Float maxErrors) {
public PhraseSuggestionBuilder maxErrors(float maxErrors) {
if (maxErrors <= 0.0) {
throw new IllegalArgumentException("max_error must be > 0.0");
}
this.maxErrors = maxErrors;
return this;
}
@ -120,6 +142,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
* not set the whitespace character is used as a separator.
*/
public PhraseSuggestionBuilder separator(String separator) {
Objects.requireNonNull(separator, "separator cannot be set to null");
this.separator = separator;
return this;
}
@ -136,13 +159,16 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
* in the dictionary. The default it <tt>0.95</tt> corresponding to 5% or
* the real words are misspelled.
*/
public PhraseSuggestionBuilder realWordErrorLikelihood(Float realWordErrorLikelihood) {
public PhraseSuggestionBuilder realWordErrorLikelihood(float realWordErrorLikelihood) {
if (realWordErrorLikelihood <= 0.0) {
throw new IllegalArgumentException("real_word_error_likelihood must be > 0.0");
}
this.realWordErrorLikelihood = realWordErrorLikelihood;
return this;
}
/**
* get the {@link #realWordErrorLikelihood(Float)} parameter
* get the {@link #realWordErrorLikelihood(float)} parameter
*/
public Float realWordErrorLikelihood() {
return this.realWordErrorLikelihood;
@ -157,7 +183,10 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
* higher than the input phrase. If set to <tt>0.0</tt> the top N candidates
* are returned. The default is <tt>1.0</tt>
*/
public PhraseSuggestionBuilder confidence(Float confidence) {
public PhraseSuggestionBuilder confidence(float confidence) {
if (confidence < 0.0) {
throw new IllegalArgumentException("confidence must be >= 0.0");
}
this.confidence = confidence;
return this;
}
@ -318,27 +347,15 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
@Override
public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
if (realWordErrorLikelihood != null) {
builder.field("real_word_error_likelihood", realWordErrorLikelihood);
}
if (confidence != null) {
builder.field("confidence", confidence);
}
if (separator != null) {
builder.field("separator", separator);
}
if (maxErrors != null) {
builder.field("max_errors", maxErrors);
}
builder.field(RWE_LIKELIHOOD_FIELD.getPreferredName(), realWordErrorLikelihood);
builder.field(CONFIDENCE_FIELD.getPreferredName(), confidence);
builder.field(SEPARATOR_FIELD.getPreferredName(), separator);
builder.field(MAXERRORS_FIELD.getPreferredName(), maxErrors);
if (gramSize != null) {
builder.field("gram_size", gramSize);
}
if (forceUnigrams != null) {
builder.field("force_unigrams", forceUnigrams);
}
if (tokenLimit != null) {
builder.field("token_limit", tokenLimit);
builder.field(GRAMSIZE_FIELD.getPreferredName(), gramSize);
}
builder.field(FORCE_UNIGRAM_FIELD.getPreferredName(), forceUnigrams);
builder.field(TOKEN_LIMIT_FIELD.getPreferredName(), tokenLimit);
if (!generators.isEmpty()) {
Set<Entry<String, List<CandidateGenerator>>> entrySet = generators.entrySet();
for (Entry<String, List<CandidateGenerator>> entry : entrySet) {
@ -350,25 +367,23 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
}
if (model != null) {
builder.startObject("smoothing");
builder.startObject(SMOOTHING_MODEL_FIELD.getPreferredName());
model.toXContent(builder, params);
builder.endObject();
}
if (preTag != null) {
builder.startObject("highlight");
builder.field("pre_tag", preTag);
builder.field("post_tag", postTag);
builder.startObject(HIGHLIGHT_FIELD.getPreferredName());
builder.field(PRE_TAG_FIELD.getPreferredName(), preTag);
builder.field(POST_TAG_FIELD.getPreferredName(), postTag);
builder.endObject();
}
if (collateQuery != null) {
builder.startObject("collate");
builder.field("query", collateQuery);
builder.startObject(COLLATE_FIELD.getPreferredName());
builder.field(COLLATE_QUERY_FIELD.getPreferredName(), collateQuery);
if (collateParams != null) {
builder.field("params", collateParams);
}
if (collatePrune != null) {
builder.field("prune", collatePrune.booleanValue());
builder.field(COLLATE_QUERY_PARAMS.getPreferredName(), collateParams);
}
builder.field(COLLATE_QUERY_PRUNE.getPreferredName(), collatePrune);
builder.endObject();
}
return builder;
@ -403,6 +418,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
private double discount = DEFAULT_BACKOFF_DISCOUNT;
private static final String NAME = "stupid_backoff";
private static final ParseField DISCOUNT_FIELD = new ParseField("discount");
private static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Creates a Stupid-Backoff smoothing model.
@ -454,7 +470,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
@Override
public SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
@ -490,6 +506,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
private double alpha = DEFAULT_LAPLACE_ALPHA;
private static final String NAME = "laplace";
private static final ParseField ALPHA_FIELD = new ParseField("alpha");
private static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Default alpha parameter for laplace smoothing
*/
@ -544,7 +561,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
@Override
public SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
@ -586,12 +603,38 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
if (obj == null || getClass() != obj.getClass()) {
return false;
}
@SuppressWarnings("unchecked")
SmoothingModel other = (SmoothingModel) obj;
return doEquals(other);
}
public abstract SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException;
public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
SmoothingModel model = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) {
model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) {
model = Laplace.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) {
model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext);
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(),
"[smoothing] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return model;
}
public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException;
public abstract WordScorerFactory buildWordScorerFactory();
@ -617,6 +660,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
private final double trigramLambda;
private final double bigramLambda;
private final double unigramLambda;
private static final ParseField PARSE_FIELD = new ParseField(NAME);
private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda");
private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda");
private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda");
@ -694,7 +738,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
@Override
public LinearInterpolation fromXContent(QueryParseContext parseContext) throws IOException {
public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
@ -726,7 +770,8 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
"suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
throw new ParsingException(parser.getTokenLocation(),
"[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda);
@ -740,6 +785,114 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
}
@Override
protected PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String suggestionName) throws IOException {
XContentParser parser = parseContext.parser();
PhraseSuggestionBuilder suggestion = new PhraseSuggestionBuilder(suggestionName);
ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
if (parseFieldMatcher.match(fieldName, SuggestionBuilder.ANALYZER_FIELD)) {
suggestion.analyzer(parser.text());
} else if (parseFieldMatcher.match(fieldName, SuggestionBuilder.FIELDNAME_FIELD)) {
suggestion.field(parser.text());
} else if (parseFieldMatcher.match(fieldName, SuggestionBuilder.SIZE_FIELD)) {
suggestion.size(parser.intValue());
} else if (parseFieldMatcher.match(fieldName, SuggestionBuilder.SHARDSIZE_FIELD)) {
suggestion.shardSize(parser.intValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.RWE_LIKELIHOOD_FIELD)) {
suggestion.realWordErrorLikelihood(parser.floatValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.CONFIDENCE_FIELD)) {
suggestion.confidence(parser.floatValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.SEPARATOR_FIELD)) {
suggestion.separator(parser.text());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.MAXERRORS_FIELD)) {
suggestion.maxErrors(parser.floatValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.GRAMSIZE_FIELD)) {
suggestion.gramSize(parser.intValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.FORCE_UNIGRAM_FIELD)) {
suggestion.forceUnigrams(parser.booleanValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.TOKEN_LIMIT_FIELD)) {
suggestion.tokenLimit(parser.intValue());
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support field [" + fieldName + "]");
}
} else if (token == Token.START_ARRAY) {
if (parseFieldMatcher.match(fieldName, DirectCandidateGeneratorBuilder.DIRECT_GENERATOR_FIELD)) {
// for now we only have a single type of generators
while ((token = parser.nextToken()) == Token.START_OBJECT) {
suggestion.addCandidateGenerator(DirectCandidateGeneratorBuilder.PROTOTYPE.fromXContent(parseContext));
}
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support array field [" + fieldName + "]");
}
} else if (token == Token.START_OBJECT) {
if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.SMOOTHING_MODEL_FIELD)) {
ensureNoSmoothing(suggestion);
suggestion.smoothingModel(SmoothingModel.fromXContent(parseContext));
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.HIGHLIGHT_FIELD)) {
String preTag = null;
String postTag = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.PRE_TAG_FIELD)) {
preTag = parser.text();
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.POST_TAG_FIELD)) {
postTag = parser.text();
} else {
throw new IllegalArgumentException(
"suggester[phrase][highlight] doesn't support field [" + fieldName + "]");
}
}
}
suggestion.highlight(preTag, postTag);
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_FIELD)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_QUERY_FIELD)) {
if (suggestion.collateQuery() != null) {
throw new IllegalArgumentException(
"suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]");
}
Template template = Template.parse(parser, parseFieldMatcher);
// TODO remember to compile script in build() method
suggestion.collateQuery(template);
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS)) {
suggestion.collateParams(parser.map());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PRUNE)) {
if (parser.isBooleanValue()) {
suggestion.collatePrune(parser.booleanValue());
} else {
throw new IllegalArgumentException("suggester[phrase][collate] prune must be either 'true' or 'false'");
}
} else {
throw new IllegalArgumentException(
"suggester[phrase][collate] doesn't support field [" + fieldName + "]");
}
}
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support array field [" + fieldName + "]");
}
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support field [" + fieldName + "]");
}
}
return suggestion;
}
private static void ensureNoSmoothing(PhraseSuggestionBuilder suggestion) {
if (suggestion.smoothingModel() != null) {
throw new IllegalArgumentException("only one smoothing model supported");
}
}
@Override
public String getWriteableName() {
return SUGGESTION_NAME;
@ -747,20 +900,20 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
@Override
public void doWriteTo(StreamOutput out) throws IOException {
out.writeOptionalFloat(maxErrors);
out.writeOptionalFloat(realWordErrorLikelihood);
out.writeOptionalFloat(confidence);
out.writeFloat(maxErrors);
out.writeFloat(realWordErrorLikelihood);
out.writeFloat(confidence);
out.writeOptionalVInt(gramSize);
boolean hasModel = model != null;
out.writeBoolean(hasModel);
if (hasModel) {
out.writePhraseSuggestionSmoothingModel(model);
}
out.writeOptionalBoolean(forceUnigrams);
out.writeOptionalVInt(tokenLimit);
out.writeBoolean(forceUnigrams);
out.writeVInt(tokenLimit);
out.writeOptionalString(preTag);
out.writeOptionalString(postTag);
out.writeOptionalString(separator);
out.writeString(separator);
if (collateQuery != null) {
out.writeBoolean(true);
collateQuery.writeTo(out);
@ -783,18 +936,18 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
@Override
public PhraseSuggestionBuilder doReadFrom(StreamInput in, String name) throws IOException {
PhraseSuggestionBuilder builder = new PhraseSuggestionBuilder(name);
builder.maxErrors = in.readOptionalFloat();
builder.realWordErrorLikelihood = in.readOptionalFloat();
builder.confidence = in.readOptionalFloat();
builder.maxErrors = in.readFloat();
builder.realWordErrorLikelihood = in.readFloat();
builder.confidence = in.readFloat();
builder.gramSize = in.readOptionalVInt();
if (in.readBoolean()) {
builder.model = in.readPhraseSuggestionSmoothingModel();
}
builder.forceUnigrams = in.readOptionalBoolean();
builder.tokenLimit = in.readOptionalVInt();
builder.forceUnigrams = in.readBoolean();
builder.tokenLimit = in.readVInt();
builder.preTag = in.readOptionalString();
builder.postTag = in.readOptionalString();
builder.separator = in.readOptionalString();
builder.separator = in.readString();
if (in.readBoolean()) {
builder.collateQuery = Template.readTemplate(in);
}

28
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
View File

@ -31,25 +31,29 @@ import java.util.List;
import java.util.Map;
class PhraseSuggestionContext extends SuggestionContext {
private final BytesRef SEPARATOR = new BytesRef(" ");
private float maxErrors = 0.5f;
private BytesRef separator = SEPARATOR;
private float realworldErrorLikelihood = 0.95f;
private List<DirectCandidateGenerator> generators = new ArrayList<>();
private int gramSize = 1;
private float confidence = 1.0f;
static final boolean DEFAULT_COLLATE_PRUNE = false;
static final boolean DEFAULT_REQUIRE_UNIGRAM = true;
static final float DEFAULT_CONFIDENCE = 1.0f;
static final int DEFAULT_GRAM_SIZE = 1;
static final float DEFAULT_RWE_ERRORLIKELIHOOD = 0.95f;
static final float DEFAULT_MAX_ERRORS = 0.5f;
static final String DEFAULT_SEPARATOR = " ";
private float maxErrors = DEFAULT_MAX_ERRORS;
private BytesRef separator = new BytesRef(DEFAULT_SEPARATOR);
private float realworldErrorLikelihood = DEFAULT_RWE_ERRORLIKELIHOOD;
private int gramSize = DEFAULT_GRAM_SIZE;
private float confidence = DEFAULT_CONFIDENCE;
private int tokenLimit = NoisyChannelSpellChecker.DEFAULT_TOKEN_LIMIT;
private boolean requireUnigram = DEFAULT_REQUIRE_UNIGRAM;
private BytesRef preTag;
private BytesRef postTag;
private CompiledScript collateQueryScript;
private CompiledScript collateFilterScript;
private boolean prune = DEFAULT_COLLATE_PRUNE;
private List<DirectCandidateGenerator> generators = new ArrayList<>();
private Map<String, Object> collateScriptParams = new HashMap<>(1);
private WordScorer.WordScorerFactory scorer;
private boolean requireUnigram = true;
private boolean prune = false;
public PhraseSuggestionContext(Suggester<? extends PhraseSuggestionContext> suggester) {
super(suggester);
}

11
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.common.text.Text;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import java.io.IOException;
@ -40,7 +41,8 @@ import java.util.List;
public final class TermSuggester extends Suggester<TermSuggestionContext> {
@Override
public TermSuggestion innerExecute(String name, TermSuggestionContext suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
public TermSuggestion innerExecute(String name, TermSuggestionContext suggestion, IndexSearcher searcher, CharsRefBuilder spare)
throws IOException {
DirectSpellChecker directSpellChecker = SuggestUtils.getDirectSpellChecker(suggestion.getDirectSpellCheckerSettings());
final IndexReader indexReader = searcher.getIndexReader();
TermSuggestion response = new TermSuggestion(
@ -76,7 +78,7 @@ public final class TermSuggester extends Suggester<TermSuggestionContext> {
@Override
public void nextToken() {
Term term = new Term(field, BytesRef.deepCopyOf(fillBytesRef(new BytesRefBuilder())));
result.add(new Token(term, offsetAttr.startOffset(), offsetAttr.endOffset()));
result.add(new Token(term, offsetAttr.startOffset(), offsetAttr.endOffset()));
}
}, spare);
return result;
@ -96,4 +98,9 @@ public final class TermSuggester extends Suggester<TermSuggestionContext> {
}
@Override
public SuggestionBuilder<?> getBuilderPrototype() {
return TermSuggestionBuilder.PROTOTYPE;
}
}

8
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import java.io.IOException;
@ -45,7 +46,7 @@ import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAUL
public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuilder> {
public static final TermSuggestionBuilder PROTOTYPE = new TermSuggestionBuilder("_na_"); // name doesn't matter
static final String SUGGESTION_NAME = "term";
private static final String SUGGESTION_NAME = "term";
private SuggestMode suggestMode = SuggestMode.MISSING;
private Float accuracy = DEFAULT_ACCURACY;
@ -341,6 +342,11 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
return builder;
}
@Override
protected TermSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String name) throws IOException {
return null;
}
@Override
public String getWriteableName() {
return SUGGESTION_NAME;

20
core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
View File

@ -110,14 +110,17 @@ public class QueryRescoreBuilderTests extends ESTestCase {
assertTrue("rescore builder is not equal to self", secondBuilder.equals(secondBuilder));
assertTrue("rescore builder is not equal to its copy", firstBuilder.equals(secondBuilder));
assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder));
assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(firstBuilder.hashCode()));
assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
equalTo(firstBuilder.hashCode()));
RescoreBuilder<?> thirdBuilder = serializedCopy(secondBuilder);
assertTrue("rescore builder is not equal to self", thirdBuilder.equals(thirdBuilder));
assertTrue("rescore builder is not equal to its copy", secondBuilder.equals(thirdBuilder));
assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
equalTo(thirdBuilder.hashCode()));
assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
assertThat("rescore builder copy's hashcode is different from original hashcode", firstBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
assertThat("rescore builder copy's hashcode is different from original hashcode", firstBuilder.hashCode(),
equalTo(thirdBuilder.hashCode()));
assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder));
assertTrue("equals is not symmetric", thirdBuilder.equals(firstBuilder));
}
@ -160,7 +163,8 @@ public class QueryRescoreBuilderTests extends ESTestCase {
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings);
// shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer
QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, null, indicesQueriesRegistry) {
QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, null,
indicesQueriesRegistry) {
@Override
public MappedFieldType fieldMapper(String name) {
StringFieldMapper.Builder builder = MapperBuilders.stringField(name);
@ -170,10 +174,11 @@ public class QueryRescoreBuilderTests extends ESTestCase {
for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
RescoreBuilder<?> rescoreBuilder = randomRescoreBuilder();
QueryRescoreContext rescoreContext = (QueryRescoreContext) rescoreBuilder.build(mockShardContext);
QueryRescoreContext rescoreContext = rescoreBuilder.build(mockShardContext);
XContentParser parser = createParser(rescoreBuilder);
QueryRescoreContext parsedRescoreContext = (QueryRescoreContext) new RescoreParseElement().parseSingleRescoreContext(parser, mockShardContext);
QueryRescoreContext parsedRescoreContext = (QueryRescoreContext) new RescoreParseElement().parseSingleRescoreContext(parser,
mockShardContext);
assertNotSame(rescoreContext, parsedRescoreContext);
assertEquals(rescoreContext.window(), parsedRescoreContext.window());
assertEquals(rescoreContext.query(), parsedRescoreContext.query());
@ -316,7 +321,8 @@ public class QueryRescoreBuilderTests extends ESTestCase {
* create random shape that is put under test
*/
public static org.elasticsearch.search.rescore.QueryRescorerBuilder randomRescoreBuilder() {
QueryBuilder<MatchAllQueryBuilder> queryBuilder = new MatchAllQueryBuilder().boost(randomFloat()).queryName(randomAsciiOfLength(20));
QueryBuilder<MatchAllQueryBuilder> queryBuilder = new MatchAllQueryBuilder().boost(randomFloat())
.queryName(randomAsciiOfLength(20));
org.elasticsearch.search.rescore.QueryRescorerBuilder rescorer = new
org.elasticsearch.search.rescore.QueryRescorerBuilder(queryBuilder);
if (randomBoolean()) {

42
core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
View File

@ -19,10 +19,19 @@
package org.elasticsearch.search.suggest;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
@ -31,6 +40,7 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import java.io.IOException;
import java.util.Collections;
import java.util.function.Consumer;
import java.util.function.Supplier;
@ -41,6 +51,7 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
private static final int NUMBER_OF_TESTBUILDERS = 20;
protected static NamedWriteableRegistry namedWriteableRegistry;
private static final Suggesters suggesters = new Suggesters(Collections.emptyMap(), null, null);
/**
* setup for the whole base test class
@ -58,6 +69,7 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
namedWriteableRegistry = null;
}
/**
* Test serialization and deserialization of the suggestion builder
*/
@ -124,6 +136,36 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
}
}
/**
* creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
*/
public void testFromXContent() throws IOException {
QueryParseContext context = new QueryParseContext(null);
context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
SB suggestionBuilder = randomTestBuilder();
XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
if (randomBoolean()) {
xContentBuilder.prettyPrint();
}
xContentBuilder.startObject();
suggestionBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
xContentBuilder.endObject();
XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes());
context.reset(parser);
// we need to skip the start object and the name, those will be parsed by outer SuggestBuilder
parser.nextToken();
parser.nextToken();
parser.nextToken();
SuggestionBuilder<?> secondSuggestionBuilder = SuggestionBuilder.fromXContent(context, suggestionBuilder.name(), suggesters);
assertNotSame(suggestionBuilder, secondSuggestionBuilder);
assertEquals(suggestionBuilder, secondSuggestionBuilder);
assertEquals(suggestionBuilder.hashCode(), secondSuggestionBuilder.hashCode());
}
}
private SB mutate(SB firstBuilder) throws IOException {
SB mutation = serializedCopy(firstBuilder);
assertNotSame(mutation, firstBuilder);

5
core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
View File

@ -71,4 +71,9 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions
this.options = options;
}
}
@Override
public SuggestionBuilder<?> getBuilderPrototype() {
return CustomSuggesterSearchIT.CustomSuggestionBuilder.PROTOTYPE;
}
}

12
core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@ -77,7 +78,9 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
assertThat(suggestions.get(1).getText().string(), is(String.format(Locale.ROOT, "%s-%s-%s-123", randomText, randomField, randomSuffix)));
}
class CustomSuggestionBuilder extends SuggestionBuilder<CustomSuggestionBuilder> {
static class CustomSuggestionBuilder extends SuggestionBuilder<CustomSuggestionBuilder> {
public final static CustomSuggestionBuilder PROTOTYPE = new CustomSuggestionBuilder("_na_", "_na_", "_na_");
private String randomField;
private String randomSuffix;
@ -122,6 +125,13 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
return Objects.hash(randomField, randomSuffix);
}
@Override
protected CustomSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String name)
throws IOException {
// TODO some parsing
return new CustomSuggestionBuilder(name, randomField, randomSuffix);
}
}
}

18
core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
View File

@ -47,8 +47,19 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
maybeSet(testBuilder::separator, randomAsciiOfLengthBetween(1, 10));
maybeSet(testBuilder::realWordErrorLikelihood, randomFloat());
maybeSet(testBuilder::confidence, randomFloat());
maybeSet(testBuilder::collatePrune, randomBoolean());
maybeSet(testBuilder::collateQuery, randomAsciiOfLengthBetween(3, 20));
// collate query prune and parameters will only be used when query is set
if (testBuilder.collateQuery() != null) {
maybeSet(testBuilder::collatePrune, randomBoolean());
if (randomBoolean()) {
Map<String, Object> collateParams = new HashMap<>();
int numParams = randomIntBetween(1, 5);
for (int i = 0; i < numParams; i++) {
collateParams.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
}
testBuilder.collateParams(collateParams );
}
}
if (randomBoolean()) {
// preTag, postTag
testBuilder.highlight(randomAsciiOfLengthBetween(3, 20), randomAsciiOfLengthBetween(3, 20));
@ -56,11 +67,6 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
maybeSet(testBuilder::gramSize, randomIntBetween(1, 5));
maybeSet(testBuilder::forceUnigrams, randomBoolean());
maybeSet(testBuilder::tokenLimit, randomInt(20));
if (randomBoolean()) {
Map<String, Object> collateParams = new HashMap<>();
collateParams.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
testBuilder.collateParams(collateParams );
}
if (randomBoolean()) {
testBuilder.smoothingModel(randomSmoothingModel());
}

54
core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
View File

@ -97,7 +97,8 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
* Test that creates new smoothing model from a random test smoothing model and checks both for equality
*/
public void testFromXContent() throws IOException {
QueryParseContext context = new QueryParseContext(new IndicesQueriesRegistry(Settings.settingsBuilder().build(), Collections.emptyMap()));
QueryParseContext context = new QueryParseContext(
new IndicesQueriesRegistry(Settings.settingsBuilder().build(), Collections.emptyMap()));
context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
SmoothingModel testModel = createTestModel();
@ -113,7 +114,7 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
parser.nextToken(); // go to start token, real parsing would do that in the outer element parser
SmoothingModel prototype = (SmoothingModel) namedWriteableRegistry.getPrototype(SmoothingModel.class,
testModel.getWriteableName());
SmoothingModel parsedModel = prototype.fromXContent(context);
SmoothingModel parsedModel = prototype.innerFromXContent(context);
assertNotSame(testModel, parsedModel);
assertEquals(testModel, parsedModel);
assertEquals(testModel.hashCode(), parsedModel.hashCode());
@ -134,7 +135,8 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
writer.addDocument(doc);
DirectoryReader ir = DirectoryReader.open(writer, false);
WordScorer wordScorer = testModel.buildWordScorerFactory().newScorer(ir, MultiFields.getTerms(ir , "field"), "field", 0.9d, BytesRefs.toBytesRef(" "));
WordScorer wordScorer = testModel.buildWordScorerFactory().newScorer(ir, MultiFields.getTerms(ir, "field"), "field", 0.9d,
BytesRefs.toBytesRef(" "));
assertWordScorer(wordScorer, testModel);
}
@ -159,35 +161,39 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
*/
@SuppressWarnings("unchecked")
public void testEqualsAndHashcode() throws IOException {
SmoothingModel firstModel = createTestModel();
assertFalse("smoothing model is equal to null", firstModel.equals(null));
assertFalse("smoothing model is equal to incompatible type", firstModel.equals(""));
assertTrue("smoothing model is not equal to self", firstModel.equals(firstModel));
assertThat("same smoothing model's hashcode returns different values if called multiple times", firstModel.hashCode(),
equalTo(firstModel.hashCode()));
assertThat("different smoothing models should not be equal", createMutation(firstModel), not(equalTo(firstModel)));
SmoothingModel firstModel = createTestModel();
assertFalse("smoothing model is equal to null", firstModel.equals(null));
assertFalse("smoothing model is equal to incompatible type", firstModel.equals(""));
assertTrue("smoothing model is not equal to self", firstModel.equals(firstModel));
assertThat("same smoothing model's hashcode returns different values if called multiple times", firstModel.hashCode(),
equalTo(firstModel.hashCode()));
assertThat("different smoothing models should not be equal", createMutation(firstModel), not(equalTo(firstModel)));
SmoothingModel secondModel = copyModel(firstModel);
assertTrue("smoothing model is not equal to self", secondModel.equals(secondModel));
assertTrue("smoothing model is not equal to its copy", firstModel.equals(secondModel));
assertTrue("equals is not symmetric", secondModel.equals(firstModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(), equalTo(firstModel.hashCode()));
SmoothingModel secondModel = copyModel(firstModel);
assertTrue("smoothing model is not equal to self", secondModel.equals(secondModel));
assertTrue("smoothing model is not equal to its copy", firstModel.equals(secondModel));
assertTrue("equals is not symmetric", secondModel.equals(firstModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(),
equalTo(firstModel.hashCode()));
SmoothingModel thirdModel = copyModel(secondModel);
assertTrue("smoothing model is not equal to self", thirdModel.equals(thirdModel));
assertTrue("smoothing model is not equal to its copy", secondModel.equals(thirdModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(), equalTo(thirdModel.hashCode()));
assertTrue("equals is not transitive", firstModel.equals(thirdModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", firstModel.hashCode(), equalTo(thirdModel.hashCode()));
assertTrue("equals is not symmetric", thirdModel.equals(secondModel));
assertTrue("equals is not symmetric", thirdModel.equals(firstModel));
SmoothingModel thirdModel = copyModel(secondModel);
assertTrue("smoothing model is not equal to self", thirdModel.equals(thirdModel));
assertTrue("smoothing model is not equal to its copy", secondModel.equals(thirdModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(),
equalTo(thirdModel.hashCode()));
assertTrue("equals is not transitive", firstModel.equals(thirdModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", firstModel.hashCode(),
equalTo(thirdModel.hashCode()));
assertTrue("equals is not symmetric", thirdModel.equals(secondModel));
assertTrue("equals is not symmetric", thirdModel.equals(firstModel));
}
static SmoothingModel copyModel(SmoothingModel original) throws IOException {
try (BytesStreamOutput output = new BytesStreamOutput()) {
original.writeTo(output);
try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
SmoothingModel prototype = (SmoothingModel) namedWriteableRegistry.getPrototype(SmoothingModel.class, original.getWriteableName());
SmoothingModel prototype = (SmoothingModel) namedWriteableRegistry.getPrototype(SmoothingModel.class,
original.getWriteableName());
return prototype.readFrom(in);
}
}

14
core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
View File

@ -20,12 +20,12 @@
package org.elasticsearch.search.suggest.term;
import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
import java.io.IOException;
import static org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy;
import static org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl;
import static org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
import static org.hamcrest.Matchers.notNullValue;
/**
@ -33,6 +33,14 @@ import static org.hamcrest.Matchers.notNullValue;
*/
public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<TermSuggestionBuilder> {
/**
* creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
*/
@Override
public void testFromXContent() throws IOException {
// skip for now
}
@Override
protected TermSuggestionBuilder randomSuggestionBuilder() {
TermSuggestionBuilder testBuilder = new TermSuggestionBuilder(randomAsciiOfLength(10));