honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Suggest: Add parsing from xContent to PhraseSuggestionBuilder 

For the ongoing search refactoring (#10217) the PhraseSuggestionBuilder
gets a way of parsing from xContent that will eventually replace the
current SuggestParseElement. This PR adds the fromXContent method
to the PhraseSuggestionBuilder and also adds parsing code for the
common suggestion parameters to SuggestionBuilder.

Also adding links from the Suggester implementations registeres in the
Suggesters registry to the corresponding prototype that is going to
be used for parsing once the refactoring is done and we switch from
parsing on shard to parsing on coordinating node.
This commit is contained in:
Christoph Büscher 2016-01-28 20:04:15 +01:00
parent 2dec129038
commit 2ae6420fd6
19 changed files with 481 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
buildSrc/src/main/resources/checkstyle_suppressions.xml
View File

@ -916,7 +916,6 @@
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]SuggestUtils.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]Suggesters.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]CompletionSuggestParser.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]CompletionSuggester.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]CategoryContextMapping.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]ContextMapping.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]context[/\\]GeoContextMapping.java" checks="LineLength" />
@ -927,12 +926,9 @@
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]LinearInterpoatingScorer.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]NoisyChannelSpellChecker.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]PhraseSuggestParser.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]PhraseSuggester.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]PhraseSuggestionBuilder.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]StupidBackoffScorer.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]WordScorer.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]term[/\\]TermSuggestParser.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]term[/\\]TermSuggester.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]RestoreService.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotInfo.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotShardFailure.java" checks="LineLength" />
@ -1446,7 +1442,6 @@
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]ExistsIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]MultiMatchQueryIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]query[/\\]SearchQueryIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]rescore[/\\]QueryRescoreBuilderTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]DuelScrollIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]SearchScrollIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]scroll[/\\]SearchScrollWithFailingNodesIT.java" checks="LineLength" />
@ -1462,7 +1457,6 @@
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]GeoContextMappingTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]DirectCandidateGeneratorTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]NoisyChannelSpellCheckerTests.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]SmoothingModelTestCase.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]similarity[/\\]SimilarityIT.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]AbstractSnapshotIntegTestCase.java" checks="LineLength" />
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]BlobStoreFormatIT.java" checks="LineLength" />

1
core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java
View File

@ -123,7 +123,6 @@ public final class SuggestParseElement implements SearchParseElement {
SuggestUtils.verifySuggestion(mapperService, globalText, suggestionContext);
suggestionSearchContext.addSuggestion(suggestionName, suggestionContext);
}
return suggestionSearchContext;
}
}

10
core/src/main/java/org/elasticsearch/search/suggest/Suggester.java
View File

@ -29,8 +29,18 @@ public abstract class Suggester<T extends SuggestionSearchContext.SuggestionCont
protected abstract Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
innerExecute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException;
/**
* link the suggester to its corresponding {@link SuggestContextParser}
* TODO: This method should eventually be removed by {@link #getBuilderPrototype()} once
* we don't directly parse from xContent to the SuggestionContext any more
*/
public abstract SuggestContextParser getContextParser();
/**
* link the suggester to its corresponding {@link SuggestionBuilder}
*/
public abstract SuggestionBuilder<?> getBuilderPrototype();
public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
execute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
// #3469 We want to ignore empty shards

12
core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java
View File

@ -64,4 +64,16 @@ public final class Suggesters extends ExtensionPoint.ClassMap<Suggester> {
public Suggester get(String type) {
return parsers.get(type);
}
public SuggestionBuilder<?> getSuggestionPrototype(String suggesterName) {
Suggester<?> suggester = parsers.get(suggesterName);
if (suggester == null) {
throw new IllegalArgumentException("suggester with name [" + suggesterName + "] not supported");
}
SuggestionBuilder<?> suggestParser = suggester.getBuilderPrototype();
if (suggestParser == null) {
throw new IllegalArgumentException("suggester with name [" + suggesterName + "] not supported");
}
return suggestParser;
}
}

55
core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
View File

@ -21,10 +21,13 @@ package org.elasticsearch.search.suggest;
import org.elasticsearch.action.support.ToXContentToBytes;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import java.io.IOException;
import java.util.Objects;
@ -138,12 +141,62 @@ public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends
return builder;
}
protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
public static SuggestionBuilder<?> fromXContent(QueryParseContext parseContext, String suggestionName, Suggesters suggesters)
throws IOException {
XContentParser parser = parseContext.parser();
ParseFieldMatcher parsefieldMatcher = parseContext.parseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
String suggestText = null;
String prefix = null;
String regex = null;
SuggestionBuilder<?> suggestionBuilder = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
if (parsefieldMatcher.match(fieldName, TEXT_FIELD)) {
suggestText = parser.text();
} else if (parsefieldMatcher.match(fieldName, PREFIX_FIELD)) {
prefix = parser.text();
} else if (parsefieldMatcher.match(fieldName, REGEX_FIELD)) {
regex = parser.text();
} else {
throw new IllegalArgumentException("[suggestion] does not support [" + fieldName + "]");
}
} else if (token == XContentParser.Token.START_OBJECT) {
if (suggestionName == null) {
throw new IllegalArgumentException("Suggestion must have name");
}
SuggestionBuilder<?> suggestParser = suggesters.getSuggestionPrototype(fieldName);
if (suggestParser == null) {
throw new IllegalArgumentException("Suggester[" + fieldName + "] not supported");
}
suggestionBuilder = suggestParser.innerFromXContent(parseContext, suggestionName);
}
}
if (suggestText != null) {
suggestionBuilder.text(suggestText);
}
if (prefix != null) {
suggestionBuilder.prefix(prefix);
}
if (regex != null) {
suggestionBuilder.regex(regex);
}
return suggestionBuilder;
}
protected abstract SuggestionBuilder<T> innerFromXContent(QueryParseContext parseContext, String name) throws IOException;
private String getSuggesterName() {
//default impl returns the same as writeable name, but we keep the distinction between the two just to make sure
return getWriteableName();
}
protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
/**
* Sets from what field to fetch the candidate suggestions from. This is an

10
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
View File

@ -38,6 +38,7 @@ import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import java.io.IOException;
import java.util.ArrayList;
@ -50,6 +51,7 @@ import java.util.Set;
public class CompletionSuggester extends Suggester<CompletionSuggestionContext> {
@Override
public SuggestContextParser getContextParser() {
return new CompletionSuggestParser(this);
}
@ -86,7 +88,8 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
for (String field : payloadFields) {
MappedFieldType payloadFieldType = suggestionContext.getMapperService().fullName(field);
if (payloadFieldType != null) {
final AtomicFieldData data = suggestionContext.getIndexFieldDataService().getForField(payloadFieldType).load(subReaderContext);
final AtomicFieldData data = suggestionContext.getIndexFieldDataService().getForField(payloadFieldType)
.load(subReaderContext);
final ScriptDocValues scriptValues = data.getScriptValues();
scriptValues.setNextDocId(subDocId);
payload.put(field, new ArrayList<>(scriptValues.getValues()));
@ -262,4 +265,9 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
}
}
}
@Override
public SuggestionBuilder<?> getBuilderPrototype() {
return CompletionSuggestionBuilder.PROTOTYPE;
}
}

8
core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
View File

@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.RegexpFlag;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
@ -50,7 +51,7 @@ import java.util.Set;
public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSuggestionBuilder> {
public static final CompletionSuggestionBuilder PROTOTYPE = new CompletionSuggestionBuilder("_na_"); // name doesn't matter
final static String SUGGESTION_NAME = "completion";
static final String SUGGESTION_NAME = "completion";
static final ParseField PAYLOAD_FIELD = new ParseField("payload");
static final ParseField CONTEXTS_FIELD = new ParseField("contexts", "context");
@ -369,6 +370,11 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
return builder;
}
@Override
protected CompletionSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String name) throws IOException {
return new CompletionSuggestionBuilder(name);
}
@Override
public String getWriteableName() {
return SUGGESTION_NAME;

30
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
View File

@ -44,6 +44,7 @@ import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result;
import java.io.IOException;
@ -65,14 +66,14 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
/*
* More Ideas:
* - add ability to find whitespace problems -> we can build a poor mans decompounder with our index based on a automaton?
* - add ability to build different error models maybe based on a confusion matrix?
* - add ability to build different error models maybe based on a confusion matrix?
* - try to combine a token with its subsequent token to find / detect word splits (optional)
* - for this to work we need some way to defined the position length of a candidate
* - phonetic filters could be interesting here too for candidate selection
*/
@Override
public Suggestion<? extends Entry<? extends Option>> innerExecute(String name, PhraseSuggestionContext suggestion, IndexSearcher searcher,
CharsRefBuilder spare) throws IOException {
public Suggestion<? extends Entry<? extends Option>> innerExecute(String name, PhraseSuggestionContext suggestion,
IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
double realWordErrorLikelihood = suggestion.realworldErrorLikelyhood();
final PhraseSuggestion response = new PhraseSuggestion(name, suggestion.getSize());
final IndexReader indexReader = searcher.getIndexReader();
@ -84,21 +85,23 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
DirectSpellChecker directSpellChecker = SuggestUtils.getDirectSpellChecker(generator);
Terms terms = MultiFields.getTerms(indexReader, generator.field());
if (terms != null) {
gens.add(new DirectCandidateGenerator(directSpellChecker, generator.field(), generator.suggestMode(),
indexReader, realWordErrorLikelihood, generator.size(), generator.preFilter(), generator.postFilter(), terms));
gens.add(new DirectCandidateGenerator(directSpellChecker, generator.field(), generator.suggestMode(),
indexReader, realWordErrorLikelihood, generator.size(), generator.preFilter(), generator.postFilter(), terms));
}
}
final String suggestField = suggestion.getField();
final Terms suggestTerms = MultiFields.getTerms(indexReader, suggestField);
if (gens.size() > 0 && suggestTerms != null) {
final NoisyChannelSpellChecker checker = new NoisyChannelSpellChecker(realWordErrorLikelihood, suggestion.getRequireUnigram(), suggestion.getTokenLimit());
final NoisyChannelSpellChecker checker = new NoisyChannelSpellChecker(realWordErrorLikelihood, suggestion.getRequireUnigram(),
suggestion.getTokenLimit());
final BytesRef separator = suggestion.separator();
WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood, separator);
WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood,
separator);
Result checkerResult;
try (TokenStream stream = checker.tokenStream(suggestion.getAnalyzer(), suggestion.getText(), spare, suggestion.getField())) {
checkerResult = checker.getCorrections(stream, new MultiCandidateGeneratorWrapper(suggestion.getShardSize(),
gens.toArray(new CandidateGenerator[gens.size()])), suggestion.maxErrors(),
suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize());
checkerResult = checker.getCorrections(stream,
new MultiCandidateGeneratorWrapper(suggestion.getShardSize(), gens.toArray(new CandidateGenerator[gens.size()])),
suggestion.maxErrors(), suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize());
}
PhraseSuggestion.Entry resultEntry = buildResultEntry(suggestion, spare, checkerResult.cutoffScore);
@ -152,10 +155,15 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
ScriptService scriptService() {
return scriptService;
}
@Override
public SuggestContextParser getContextParser() {
return new PhraseSuggestParser(this);
}
@Override
public SuggestionBuilder<?> getBuilderPrototype() {
return PhraseSuggestionBuilder.PROTOTYPE;
}
}

275
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.search.suggest.phrase;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.BytesRef;
@ -51,24 +52,42 @@ import java.util.Set;
*/
public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSuggestionBuilder> {
static final String SUGGESTION_NAME = "phrase";
private static final String SUGGESTION_NAME = "phrase";
public static final PhraseSuggestionBuilder PROTOTYPE = new PhraseSuggestionBuilder("_na_");
private Float maxErrors;
private String separator;
private Float realWordErrorLikelihood;
private Float confidence;
private final Map<String, List<CandidateGenerator>> generators = new HashMap<>();
protected static final ParseField MAXERRORS_FIELD = new ParseField("max_errors");
protected static final ParseField RWE_LIKELIHOOD_FIELD = new ParseField("real_word_error_likelihood");
protected static final ParseField SEPARATOR_FIELD = new ParseField("separator");
protected static final ParseField CONFIDENCE_FIELD = new ParseField("confidence");
protected static final ParseField GENERATORS_FIELD = new ParseField("shard_size");
protected static final ParseField GRAMSIZE_FIELD = new ParseField("gram_size");
protected static final ParseField SMOOTHING_MODEL_FIELD = new ParseField("smoothing");
protected static final ParseField FORCE_UNIGRAM_FIELD = new ParseField("force_unigrams");
protected static final ParseField TOKEN_LIMIT_FIELD = new ParseField("token_limit");
protected static final ParseField HIGHLIGHT_FIELD = new ParseField("highlight");
protected static final ParseField PRE_TAG_FIELD = new ParseField("pre_tag");
protected static final ParseField POST_TAG_FIELD = new ParseField("post_tag");
protected static final ParseField COLLATE_FIELD = new ParseField("collate");
protected static final ParseField COLLATE_QUERY_FIELD = new ParseField("query");
protected static final ParseField COLLATE_QUERY_PARAMS = new ParseField("params");
protected static final ParseField COLLATE_QUERY_PRUNE = new ParseField("prune");
private float maxErrors = PhraseSuggestionContext.DEFAULT_MAX_ERRORS;
private String separator = PhraseSuggestionContext.DEFAULT_SEPARATOR;
private float realWordErrorLikelihood = PhraseSuggestionContext.DEFAULT_RWE_ERRORLIKELIHOOD;
private float confidence = PhraseSuggestionContext.DEFAULT_CONFIDENCE;
// gramSize needs to be optional although there is a default, if unset parser try to detect and use shingle size
private Integer gramSize;
private SmoothingModel model;
private Boolean forceUnigrams;
private Integer tokenLimit;
private boolean forceUnigrams = PhraseSuggestionContext.DEFAULT_REQUIRE_UNIGRAM;
private int tokenLimit = NoisyChannelSpellChecker.DEFAULT_TOKEN_LIMIT;
private String preTag;
private String postTag;
private Template collateQuery;
private Map<String, Object> collateParams;
private Boolean collatePrune;
private boolean collatePrune = PhraseSuggestionContext.DEFAULT_COLLATE_PRUNE;
private SmoothingModel model;
private final Map<String, List<CandidateGenerator>> generators = new HashMap<>();
public PhraseSuggestionBuilder(String name) {
super(name);
@ -103,7 +122,10 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
* The default is set to <tt>1.0</tt> which corresponds to that only
* corrections with at most 1 missspelled term are returned.
*/
public PhraseSuggestionBuilder maxErrors(Float maxErrors) {
public PhraseSuggestionBuilder maxErrors(float maxErrors) {
if (maxErrors <= 0.0) {
throw new IllegalArgumentException("max_error must be > 0.0");
}
this.maxErrors = maxErrors;
return this;
}
@ -120,6 +142,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
* not set the whitespace character is used as a separator.
*/
public PhraseSuggestionBuilder separator(String separator) {
Objects.requireNonNull(separator, "separator cannot be set to null");
this.separator = separator;
return this;
}
@ -136,13 +159,16 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
* in the dictionary. The default it <tt>0.95</tt> corresponding to 5% or
* the real words are misspelled.
*/
public PhraseSuggestionBuilder realWordErrorLikelihood(Float realWordErrorLikelihood) {
public PhraseSuggestionBuilder realWordErrorLikelihood(float realWordErrorLikelihood) {
if (realWordErrorLikelihood <= 0.0) {
throw new IllegalArgumentException("real_word_error_likelihood must be > 0.0");
}
this.realWordErrorLikelihood = realWordErrorLikelihood;
return this;
}
/**
* get the {@link #realWordErrorLikelihood(Float)} parameter
* get the {@link #realWordErrorLikelihood(float)} parameter
*/
public Float realWordErrorLikelihood() {
return this.realWordErrorLikelihood;
@ -157,7 +183,10 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
* higher than the input phrase. If set to <tt>0.0</tt> the top N candidates
* are returned. The default is <tt>1.0</tt>
*/
public PhraseSuggestionBuilder confidence(Float confidence) {
public PhraseSuggestionBuilder confidence(float confidence) {
if (confidence < 0.0) {
throw new IllegalArgumentException("confidence must be >= 0.0");
}
this.confidence = confidence;
return this;
}
@ -318,27 +347,15 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
@Override
public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
if (realWordErrorLikelihood != null) {
builder.field("real_word_error_likelihood", realWordErrorLikelihood);
}
if (confidence != null) {
builder.field("confidence", confidence);
}
if (separator != null) {
builder.field("separator", separator);
}
if (maxErrors != null) {
builder.field("max_errors", maxErrors);
}
builder.field(RWE_LIKELIHOOD_FIELD.getPreferredName(), realWordErrorLikelihood);
builder.field(CONFIDENCE_FIELD.getPreferredName(), confidence);
builder.field(SEPARATOR_FIELD.getPreferredName(), separator);
builder.field(MAXERRORS_FIELD.getPreferredName(), maxErrors);
if (gramSize != null) {
builder.field("gram_size", gramSize);
}
if (forceUnigrams != null) {
builder.field("force_unigrams", forceUnigrams);
}
if (tokenLimit != null) {
builder.field("token_limit", tokenLimit);
builder.field(GRAMSIZE_FIELD.getPreferredName(), gramSize);
}
builder.field(FORCE_UNIGRAM_FIELD.getPreferredName(), forceUnigrams);
builder.field(TOKEN_LIMIT_FIELD.getPreferredName(), tokenLimit);
if (!generators.isEmpty()) {
Set<Entry<String, List<CandidateGenerator>>> entrySet = generators.entrySet();
for (Entry<String, List<CandidateGenerator>> entry : entrySet) {
@ -350,25 +367,23 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
}
if (model != null) {
builder.startObject("smoothing");
builder.startObject(SMOOTHING_MODEL_FIELD.getPreferredName());
model.toXContent(builder, params);
builder.endObject();
}
if (preTag != null) {
builder.startObject("highlight");
builder.field("pre_tag", preTag);
builder.field("post_tag", postTag);
builder.startObject(HIGHLIGHT_FIELD.getPreferredName());
builder.field(PRE_TAG_FIELD.getPreferredName(), preTag);
builder.field(POST_TAG_FIELD.getPreferredName(), postTag);
builder.endObject();
}
if (collateQuery != null) {
builder.startObject("collate");
builder.field("query", collateQuery);
builder.startObject(COLLATE_FIELD.getPreferredName());
builder.field(COLLATE_QUERY_FIELD.getPreferredName(), collateQuery);
if (collateParams != null) {
builder.field("params", collateParams);
}
if (collatePrune != null) {
builder.field("prune", collatePrune.booleanValue());
builder.field(COLLATE_QUERY_PARAMS.getPreferredName(), collateParams);
}
builder.field(COLLATE_QUERY_PRUNE.getPreferredName(), collatePrune);
builder.endObject();
}
return builder;
@ -403,6 +418,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
private double discount = DEFAULT_BACKOFF_DISCOUNT;
private static final String NAME = "stupid_backoff";
private static final ParseField DISCOUNT_FIELD = new ParseField("discount");
private static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Creates a Stupid-Backoff smoothing model.
@ -454,7 +470,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
@Override
public SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
@ -490,6 +506,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
private double alpha = DEFAULT_LAPLACE_ALPHA;
private static final String NAME = "laplace";
private static final ParseField ALPHA_FIELD = new ParseField("alpha");
private static final ParseField PARSE_FIELD = new ParseField(NAME);
/**
* Default alpha parameter for laplace smoothing
*/
@ -544,7 +561,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
@Override
public SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
@ -586,12 +603,38 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
if (obj == null || getClass() != obj.getClass()) {
return false;
}
@SuppressWarnings("unchecked")
SmoothingModel other = (SmoothingModel) obj;
return doEquals(other);
}
public abstract SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException;
public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
SmoothingModel model = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) {
model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) {
model = Laplace.PROTOTYPE.innerFromXContent(parseContext);
} else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) {
model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext);
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(),
"[smoothing] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return model;
}
public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException;
public abstract WordScorerFactory buildWordScorerFactory();
@ -617,6 +660,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
private final double trigramLambda;
private final double bigramLambda;
private final double unigramLambda;
private static final ParseField PARSE_FIELD = new ParseField(NAME);
private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda");
private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda");
private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda");
@ -694,7 +738,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
@Override
public LinearInterpolation fromXContent(QueryParseContext parseContext) throws IOException {
public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException {
XContentParser parser = parseContext.parser();
XContentParser.Token token;
String fieldName = null;
@ -726,7 +770,8 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
"suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]");
}
} else {
throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
throw new ParsingException(parser.getTokenLocation(),
"[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
}
}
return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda);
@ -740,6 +785,114 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
}
}
@Override
protected PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String suggestionName) throws IOException {
XContentParser parser = parseContext.parser();
PhraseSuggestionBuilder suggestion = new PhraseSuggestionBuilder(suggestionName);
ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
XContentParser.Token token;
String fieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
if (parseFieldMatcher.match(fieldName, SuggestionBuilder.ANALYZER_FIELD)) {
suggestion.analyzer(parser.text());
} else if (parseFieldMatcher.match(fieldName, SuggestionBuilder.FIELDNAME_FIELD)) {
suggestion.field(parser.text());
} else if (parseFieldMatcher.match(fieldName, SuggestionBuilder.SIZE_FIELD)) {
suggestion.size(parser.intValue());
} else if (parseFieldMatcher.match(fieldName, SuggestionBuilder.SHARDSIZE_FIELD)) {
suggestion.shardSize(parser.intValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.RWE_LIKELIHOOD_FIELD)) {
suggestion.realWordErrorLikelihood(parser.floatValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.CONFIDENCE_FIELD)) {
suggestion.confidence(parser.floatValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.SEPARATOR_FIELD)) {
suggestion.separator(parser.text());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.MAXERRORS_FIELD)) {
suggestion.maxErrors(parser.floatValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.GRAMSIZE_FIELD)) {
suggestion.gramSize(parser.intValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.FORCE_UNIGRAM_FIELD)) {
suggestion.forceUnigrams(parser.booleanValue());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.TOKEN_LIMIT_FIELD)) {
suggestion.tokenLimit(parser.intValue());
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support field [" + fieldName + "]");
}
} else if (token == Token.START_ARRAY) {
if (parseFieldMatcher.match(fieldName, DirectCandidateGeneratorBuilder.DIRECT_GENERATOR_FIELD)) {
// for now we only have a single type of generators
while ((token = parser.nextToken()) == Token.START_OBJECT) {
suggestion.addCandidateGenerator(DirectCandidateGeneratorBuilder.PROTOTYPE.fromXContent(parseContext));
}
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support array field [" + fieldName + "]");
}
} else if (token == Token.START_OBJECT) {
if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.SMOOTHING_MODEL_FIELD)) {
ensureNoSmoothing(suggestion);
suggestion.smoothingModel(SmoothingModel.fromXContent(parseContext));
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.HIGHLIGHT_FIELD)) {
String preTag = null;
String postTag = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.PRE_TAG_FIELD)) {
preTag = parser.text();
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.POST_TAG_FIELD)) {
postTag = parser.text();
} else {
throw new IllegalArgumentException(
"suggester[phrase][highlight] doesn't support field [" + fieldName + "]");
}
}
}
suggestion.highlight(preTag, postTag);
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_FIELD)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_QUERY_FIELD)) {
if (suggestion.collateQuery() != null) {
throw new IllegalArgumentException(
"suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]");
}
Template template = Template.parse(parser, parseFieldMatcher);
// TODO remember to compile script in build() method
suggestion.collateQuery(template);
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS)) {
suggestion.collateParams(parser.map());
} else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PRUNE)) {
if (parser.isBooleanValue()) {
suggestion.collatePrune(parser.booleanValue());
} else {
throw new IllegalArgumentException("suggester[phrase][collate] prune must be either 'true' or 'false'");
}
} else {
throw new IllegalArgumentException(
"suggester[phrase][collate] doesn't support field [" + fieldName + "]");
}
}
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support array field [" + fieldName + "]");
}
} else {
throw new IllegalArgumentException("suggester[phrase] doesn't support field [" + fieldName + "]");
}
}
return suggestion;
}
private static void ensureNoSmoothing(PhraseSuggestionBuilder suggestion) {
if (suggestion.smoothingModel() != null) {
throw new IllegalArgumentException("only one smoothing model supported");
}
}
@Override
public String getWriteableName() {
return SUGGESTION_NAME;
@ -747,20 +900,20 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
@Override
public void doWriteTo(StreamOutput out) throws IOException {
out.writeOptionalFloat(maxErrors);
out.writeOptionalFloat(realWordErrorLikelihood);
out.writeOptionalFloat(confidence);
out.writeFloat(maxErrors);
out.writeFloat(realWordErrorLikelihood);
out.writeFloat(confidence);
out.writeOptionalVInt(gramSize);
boolean hasModel = model != null;
out.writeBoolean(hasModel);
if (hasModel) {
out.writePhraseSuggestionSmoothingModel(model);
}
out.writeOptionalBoolean(forceUnigrams);
out.writeOptionalVInt(tokenLimit);
out.writeBoolean(forceUnigrams);
out.writeVInt(tokenLimit);
out.writeOptionalString(preTag);
out.writeOptionalString(postTag);
out.writeOptionalString(separator);
out.writeString(separator);
if (collateQuery != null) {
out.writeBoolean(true);
collateQuery.writeTo(out);
@ -783,18 +936,18 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
@Override
public PhraseSuggestionBuilder doReadFrom(StreamInput in, String name) throws IOException {
PhraseSuggestionBuilder builder = new PhraseSuggestionBuilder(name);
builder.maxErrors = in.readOptionalFloat();
builder.realWordErrorLikelihood = in.readOptionalFloat();
builder.confidence = in.readOptionalFloat();
builder.maxErrors = in.readFloat();
builder.realWordErrorLikelihood = in.readFloat();
builder.confidence = in.readFloat();
builder.gramSize = in.readOptionalVInt();
if (in.readBoolean()) {
builder.model = in.readPhraseSuggestionSmoothingModel();
}
builder.forceUnigrams = in.readOptionalBoolean();
builder.tokenLimit = in.readOptionalVInt();
builder.forceUnigrams = in.readBoolean();
builder.tokenLimit = in.readVInt();
builder.preTag = in.readOptionalString();
builder.postTag = in.readOptionalString();
builder.separator = in.readOptionalString();
builder.separator = in.readString();
if (in.readBoolean()) {
builder.collateQuery = Template.readTemplate(in);
}

28
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
View File

@ -31,25 +31,29 @@ import java.util.List;
import java.util.Map;
class PhraseSuggestionContext extends SuggestionContext {
private final BytesRef SEPARATOR = new BytesRef(" ");
private float maxErrors = 0.5f;
private BytesRef separator = SEPARATOR;
private float realworldErrorLikelihood = 0.95f;
private List<DirectCandidateGenerator> generators = new ArrayList<>();
private int gramSize = 1;
private float confidence = 1.0f;
static final boolean DEFAULT_COLLATE_PRUNE = false;
static final boolean DEFAULT_REQUIRE_UNIGRAM = true;
static final float DEFAULT_CONFIDENCE = 1.0f;
static final int DEFAULT_GRAM_SIZE = 1;
static final float DEFAULT_RWE_ERRORLIKELIHOOD = 0.95f;
static final float DEFAULT_MAX_ERRORS = 0.5f;
static final String DEFAULT_SEPARATOR = " ";
private float maxErrors = DEFAULT_MAX_ERRORS;
private BytesRef separator = new BytesRef(DEFAULT_SEPARATOR);
private float realworldErrorLikelihood = DEFAULT_RWE_ERRORLIKELIHOOD;
private int gramSize = DEFAULT_GRAM_SIZE;
private float confidence = DEFAULT_CONFIDENCE;
private int tokenLimit = NoisyChannelSpellChecker.DEFAULT_TOKEN_LIMIT;
private boolean requireUnigram = DEFAULT_REQUIRE_UNIGRAM;
private BytesRef preTag;
private BytesRef postTag;
private CompiledScript collateQueryScript;
private CompiledScript collateFilterScript;
private boolean prune = DEFAULT_COLLATE_PRUNE;
private List<DirectCandidateGenerator> generators = new ArrayList<>();
private Map<String, Object> collateScriptParams = new HashMap<>(1);
private WordScorer.WordScorerFactory scorer;
private boolean requireUnigram = true;
private boolean prune = false;
public PhraseSuggestionContext(Suggester<? extends PhraseSuggestionContext> suggester) {
super(suggester);
}

11
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.common.text.Text;
import org.elasticsearch.search.suggest.SuggestContextParser;
import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import java.io.IOException;
@ -40,7 +41,8 @@ import java.util.List;
public final class TermSuggester extends Suggester<TermSuggestionContext> {
@Override
public TermSuggestion innerExecute(String name, TermSuggestionContext suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
public TermSuggestion innerExecute(String name, TermSuggestionContext suggestion, IndexSearcher searcher, CharsRefBuilder spare)
throws IOException {
DirectSpellChecker directSpellChecker = SuggestUtils.getDirectSpellChecker(suggestion.getDirectSpellCheckerSettings());
final IndexReader indexReader = searcher.getIndexReader();
TermSuggestion response = new TermSuggestion(
@ -76,7 +78,7 @@ public final class TermSuggester extends Suggester<TermSuggestionContext> {
@Override
public void nextToken() {
Term term = new Term(field, BytesRef.deepCopyOf(fillBytesRef(new BytesRefBuilder())));
result.add(new Token(term, offsetAttr.startOffset(), offsetAttr.endOffset()));
result.add(new Token(term, offsetAttr.startOffset(), offsetAttr.endOffset()));
}
}, spare);
return result;
@ -96,4 +98,9 @@ public final class TermSuggester extends Suggester<TermSuggestionContext> {
}
@Override
public SuggestionBuilder<?> getBuilderPrototype() {
return TermSuggestionBuilder.PROTOTYPE;
}
}

8
core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import java.io.IOException;
@ -45,7 +46,7 @@ import static org.elasticsearch.search.suggest.DirectSpellcheckerSettings.DEFAUL
public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuilder> {
public static final TermSuggestionBuilder PROTOTYPE = new TermSuggestionBuilder("_na_"); // name doesn't matter
static final String SUGGESTION_NAME = "term";
private static final String SUGGESTION_NAME = "term";
private SuggestMode suggestMode = SuggestMode.MISSING;
private Float accuracy = DEFAULT_ACCURACY;
@ -341,6 +342,11 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
return builder;
}
@Override
protected TermSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String name) throws IOException {
return null;
}
@Override
public String getWriteableName() {
return SUGGESTION_NAME;

20
core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
View File

@ -110,14 +110,17 @@ public class QueryRescoreBuilderTests extends ESTestCase {
assertTrue("rescore builder is not equal to self", secondBuilder.equals(secondBuilder));
assertTrue("rescore builder is not equal to its copy", firstBuilder.equals(secondBuilder));
assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder));
assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(firstBuilder.hashCode()));
assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
equalTo(firstBuilder.hashCode()));
RescoreBuilder<?> thirdBuilder = serializedCopy(secondBuilder);
assertTrue("rescore builder is not equal to self", thirdBuilder.equals(thirdBuilder));
assertTrue("rescore builder is not equal to its copy", secondBuilder.equals(thirdBuilder));
assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
assertThat("rescore builder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
equalTo(thirdBuilder.hashCode()));
assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
assertThat("rescore builder copy's hashcode is different from original hashcode", firstBuilder.hashCode(), equalTo(thirdBuilder.hashCode()));
assertThat("rescore builder copy's hashcode is different from original hashcode", firstBuilder.hashCode(),
equalTo(thirdBuilder.hashCode()));
assertTrue("equals is not symmetric", thirdBuilder.equals(secondBuilder));
assertTrue("equals is not symmetric", thirdBuilder.equals(firstBuilder));
}
@ -160,7 +163,8 @@ public class QueryRescoreBuilderTests extends ESTestCase {
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings);
// shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer
QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, null, indicesQueriesRegistry) {
QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, null,
indicesQueriesRegistry) {
@Override
public MappedFieldType fieldMapper(String name) {
StringFieldMapper.Builder builder = MapperBuilders.stringField(name);
@ -170,10 +174,11 @@ public class QueryRescoreBuilderTests extends ESTestCase {
for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
RescoreBuilder<?> rescoreBuilder = randomRescoreBuilder();
QueryRescoreContext rescoreContext = (QueryRescoreContext) rescoreBuilder.build(mockShardContext);
QueryRescoreContext rescoreContext = rescoreBuilder.build(mockShardContext);
XContentParser parser = createParser(rescoreBuilder);
QueryRescoreContext parsedRescoreContext = (QueryRescoreContext) new RescoreParseElement().parseSingleRescoreContext(parser, mockShardContext);
QueryRescoreContext parsedRescoreContext = (QueryRescoreContext) new RescoreParseElement().parseSingleRescoreContext(parser,
mockShardContext);
assertNotSame(rescoreContext, parsedRescoreContext);
assertEquals(rescoreContext.window(), parsedRescoreContext.window());
assertEquals(rescoreContext.query(), parsedRescoreContext.query());
@ -316,7 +321,8 @@ public class QueryRescoreBuilderTests extends ESTestCase {
* create random shape that is put under test
*/
public static org.elasticsearch.search.rescore.QueryRescorerBuilder randomRescoreBuilder() {
QueryBuilder<MatchAllQueryBuilder> queryBuilder = new MatchAllQueryBuilder().boost(randomFloat()).queryName(randomAsciiOfLength(20));
QueryBuilder<MatchAllQueryBuilder> queryBuilder = new MatchAllQueryBuilder().boost(randomFloat())
.queryName(randomAsciiOfLength(20));
org.elasticsearch.search.rescore.QueryRescorerBuilder rescorer = new
org.elasticsearch.search.rescore.QueryRescorerBuilder(queryBuilder);
if (randomBoolean()) {

42
core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
View File

@ -19,10 +19,19 @@
package org.elasticsearch.search.suggest;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
@ -31,6 +40,7 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import java.io.IOException;
import java.util.Collections;
import java.util.function.Consumer;
import java.util.function.Supplier;
@ -41,6 +51,7 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
private static final int NUMBER_OF_TESTBUILDERS = 20;
protected static NamedWriteableRegistry namedWriteableRegistry;
private static final Suggesters suggesters = new Suggesters(Collections.emptyMap(), null, null);
/**
* setup for the whole base test class
@ -58,6 +69,7 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
namedWriteableRegistry = null;
}
/**
* Test serialization and deserialization of the suggestion builder
*/
@ -124,6 +136,36 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
}
}
/**
* creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
*/
public void testFromXContent() throws IOException {
QueryParseContext context = new QueryParseContext(null);
context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
SB suggestionBuilder = randomTestBuilder();
XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
if (randomBoolean()) {
xContentBuilder.prettyPrint();
}
xContentBuilder.startObject();
suggestionBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
xContentBuilder.endObject();
XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes());
context.reset(parser);
// we need to skip the start object and the name, those will be parsed by outer SuggestBuilder
parser.nextToken();
parser.nextToken();
parser.nextToken();
SuggestionBuilder<?> secondSuggestionBuilder = SuggestionBuilder.fromXContent(context, suggestionBuilder.name(), suggesters);
assertNotSame(suggestionBuilder, secondSuggestionBuilder);
assertEquals(suggestionBuilder, secondSuggestionBuilder);
assertEquals(suggestionBuilder.hashCode(), secondSuggestionBuilder.hashCode());
}
}
private SB mutate(SB firstBuilder) throws IOException {
SB mutation = serializedCopy(firstBuilder);
assertNotSame(mutation, firstBuilder);

5
core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
View File

@ -71,4 +71,9 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions
this.options = options;
}
}
@Override
public SuggestionBuilder<?> getBuilderPrototype() {
return CustomSuggesterSearchIT.CustomSuggestionBuilder.PROTOTYPE;
}
}

12
core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@ -77,7 +78,9 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
assertThat(suggestions.get(1).getText().string(), is(String.format(Locale.ROOT, "%s-%s-%s-123", randomText, randomField, randomSuffix)));
}
class CustomSuggestionBuilder extends SuggestionBuilder<CustomSuggestionBuilder> {
static class CustomSuggestionBuilder extends SuggestionBuilder<CustomSuggestionBuilder> {
public final static CustomSuggestionBuilder PROTOTYPE = new CustomSuggestionBuilder("_na_", "_na_", "_na_");
private String randomField;
private String randomSuffix;
@ -122,6 +125,13 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
return Objects.hash(randomField, randomSuffix);
}
@Override
protected CustomSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String name)
throws IOException {
// TODO some parsing
return new CustomSuggestionBuilder(name, randomField, randomSuffix);
}
}
}

18
core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
View File

@ -47,8 +47,19 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
maybeSet(testBuilder::separator, randomAsciiOfLengthBetween(1, 10));
maybeSet(testBuilder::realWordErrorLikelihood, randomFloat());
maybeSet(testBuilder::confidence, randomFloat());
maybeSet(testBuilder::collatePrune, randomBoolean());
maybeSet(testBuilder::collateQuery, randomAsciiOfLengthBetween(3, 20));
// collate query prune and parameters will only be used when query is set
if (testBuilder.collateQuery() != null) {
maybeSet(testBuilder::collatePrune, randomBoolean());
if (randomBoolean()) {
Map<String, Object> collateParams = new HashMap<>();
int numParams = randomIntBetween(1, 5);
for (int i = 0; i < numParams; i++) {
collateParams.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
}
testBuilder.collateParams(collateParams );
}
}
if (randomBoolean()) {
// preTag, postTag
testBuilder.highlight(randomAsciiOfLengthBetween(3, 20), randomAsciiOfLengthBetween(3, 20));
@ -56,11 +67,6 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
maybeSet(testBuilder::gramSize, randomIntBetween(1, 5));
maybeSet(testBuilder::forceUnigrams, randomBoolean());
maybeSet(testBuilder::tokenLimit, randomInt(20));
if (randomBoolean()) {
Map<String, Object> collateParams = new HashMap<>();
collateParams.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
testBuilder.collateParams(collateParams );
}
if (randomBoolean()) {
testBuilder.smoothingModel(randomSmoothingModel());
}

54
core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
View File

@ -97,7 +97,8 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
* Test that creates new smoothing model from a random test smoothing model and checks both for equality
*/
public void testFromXContent() throws IOException {
QueryParseContext context = new QueryParseContext(new IndicesQueriesRegistry(Settings.settingsBuilder().build(), Collections.emptyMap()));
QueryParseContext context = new QueryParseContext(
new IndicesQueriesRegistry(Settings.settingsBuilder().build(), Collections.emptyMap()));
context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
SmoothingModel testModel = createTestModel();
@ -113,7 +114,7 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
parser.nextToken(); // go to start token, real parsing would do that in the outer element parser
SmoothingModel prototype = (SmoothingModel) namedWriteableRegistry.getPrototype(SmoothingModel.class,
testModel.getWriteableName());
SmoothingModel parsedModel = prototype.fromXContent(context);
SmoothingModel parsedModel = prototype.innerFromXContent(context);
assertNotSame(testModel, parsedModel);
assertEquals(testModel, parsedModel);
assertEquals(testModel.hashCode(), parsedModel.hashCode());
@ -134,7 +135,8 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
writer.addDocument(doc);
DirectoryReader ir = DirectoryReader.open(writer, false);
WordScorer wordScorer = testModel.buildWordScorerFactory().newScorer(ir, MultiFields.getTerms(ir , "field"), "field", 0.9d, BytesRefs.toBytesRef(" "));
WordScorer wordScorer = testModel.buildWordScorerFactory().newScorer(ir, MultiFields.getTerms(ir, "field"), "field", 0.9d,
BytesRefs.toBytesRef(" "));
assertWordScorer(wordScorer, testModel);
}
@ -159,35 +161,39 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
*/
@SuppressWarnings("unchecked")
public void testEqualsAndHashcode() throws IOException {
SmoothingModel firstModel = createTestModel();
assertFalse("smoothing model is equal to null", firstModel.equals(null));
assertFalse("smoothing model is equal to incompatible type", firstModel.equals(""));
assertTrue("smoothing model is not equal to self", firstModel.equals(firstModel));
assertThat("same smoothing model's hashcode returns different values if called multiple times", firstModel.hashCode(),
equalTo(firstModel.hashCode()));
assertThat("different smoothing models should not be equal", createMutation(firstModel), not(equalTo(firstModel)));
SmoothingModel firstModel = createTestModel();
assertFalse("smoothing model is equal to null", firstModel.equals(null));
assertFalse("smoothing model is equal to incompatible type", firstModel.equals(""));
assertTrue("smoothing model is not equal to self", firstModel.equals(firstModel));
assertThat("same smoothing model's hashcode returns different values if called multiple times", firstModel.hashCode(),
equalTo(firstModel.hashCode()));
assertThat("different smoothing models should not be equal", createMutation(firstModel), not(equalTo(firstModel)));
SmoothingModel secondModel = copyModel(firstModel);
assertTrue("smoothing model is not equal to self", secondModel.equals(secondModel));
assertTrue("smoothing model is not equal to its copy", firstModel.equals(secondModel));
assertTrue("equals is not symmetric", secondModel.equals(firstModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(), equalTo(firstModel.hashCode()));
SmoothingModel secondModel = copyModel(firstModel);
assertTrue("smoothing model is not equal to self", secondModel.equals(secondModel));
assertTrue("smoothing model is not equal to its copy", firstModel.equals(secondModel));
assertTrue("equals is not symmetric", secondModel.equals(firstModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(),
equalTo(firstModel.hashCode()));
SmoothingModel thirdModel = copyModel(secondModel);
assertTrue("smoothing model is not equal to self", thirdModel.equals(thirdModel));
assertTrue("smoothing model is not equal to its copy", secondModel.equals(thirdModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(), equalTo(thirdModel.hashCode()));
assertTrue("equals is not transitive", firstModel.equals(thirdModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", firstModel.hashCode(), equalTo(thirdModel.hashCode()));
assertTrue("equals is not symmetric", thirdModel.equals(secondModel));
assertTrue("equals is not symmetric", thirdModel.equals(firstModel));
SmoothingModel thirdModel = copyModel(secondModel);
assertTrue("smoothing model is not equal to self", thirdModel.equals(thirdModel));
assertTrue("smoothing model is not equal to its copy", secondModel.equals(thirdModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", secondModel.hashCode(),
equalTo(thirdModel.hashCode()));
assertTrue("equals is not transitive", firstModel.equals(thirdModel));
assertThat("smoothing model copy's hashcode is different from original hashcode", firstModel.hashCode(),
equalTo(thirdModel.hashCode()));
assertTrue("equals is not symmetric", thirdModel.equals(secondModel));
assertTrue("equals is not symmetric", thirdModel.equals(firstModel));
}
static SmoothingModel copyModel(SmoothingModel original) throws IOException {
try (BytesStreamOutput output = new BytesStreamOutput()) {
original.writeTo(output);
try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
SmoothingModel prototype = (SmoothingModel) namedWriteableRegistry.getPrototype(SmoothingModel.class, original.getWriteableName());
SmoothingModel prototype = (SmoothingModel) namedWriteableRegistry.getPrototype(SmoothingModel.class,
original.getWriteableName());
return prototype.readFrom(in);
}
}

14
core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
View File

@ -20,12 +20,12 @@
package org.elasticsearch.search.suggest.term;
import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
import java.io.IOException;
import static org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy;
import static org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl;
import static org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
import static org.hamcrest.Matchers.notNullValue;
/**
@ -33,6 +33,14 @@ import static org.hamcrest.Matchers.notNullValue;
*/
public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<TermSuggestionBuilder> {
/**
* creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
*/
@Override
public void testFromXContent() throws IOException {
// skip for now
}
@Override
protected TermSuggestionBuilder randomSuggestionBuilder() {
TermSuggestionBuilder testBuilder = new TermSuggestionBuilder(randomAsciiOfLength(10));