honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support fuzzy queries in CompletionSuggest 

Added the FuzzySuggester in order to support completion queries

The following options have been added for the fuxxy suggester

* edit_distance: Maximum edit distance
* transpositions: Sets if transpositions should be counted as one or two changes
* min_prefix_len: Minimum length of the input before fuzzy suggestions are returned
* non_prefix_len: Minimum length of the input, which is not checked for fuzzy alternatives

Closes #3465
This commit is contained in:
Alexander Reelsen 2013-08-12 13:48:55 +02:00
parent a7b643305a
commit f58f165522
10 changed files with 545 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

226
src/main/java/org/apache/lucene/search/suggest/analyzing/XFuzzySuggester.java Normal file
View File

@ -0,0 +1,226 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lucene.search.suggest.analyzing;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.automaton.*;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.PairOutputs;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
/**
* Implements a fuzzy {@link AnalyzingSuggester}. The similarity measurement is
* based on the Damerau-Levenshtein (optimal string alignment) algorithm, though
* you can explicitly choose classic Levenshtein by passing <code>false</code>
* for the <code>transpositions</code> parameter.
* <p>
* At most, this query will match terms up to
* {@value org.apache.lucene.util.automaton.LevenshteinAutomata#MAXIMUM_SUPPORTED_DISTANCE}
* edits. Higher distances are not supported. Note that the
* fuzzy distance is measured in "byte space" on the bytes
* returned by the {@link org.apache.lucene.analysis.TokenStream}'s {@link
* org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute}, usually UTF8. By default
* the analyzed bytes must be at least 3 {@link
* #DEFAULT_MIN_FUZZY_LENGTH} bytes before any edits are
* considered. Furthermore, the first 1 {@link
* #DEFAULT_NON_FUZZY_PREFIX} byte is not allowed to be
* edited. We allow up to 1 (@link
* #DEFAULT_MAX_EDITS} edit.
*
* <p>
* NOTE: This suggester does not boost suggestions that
* required no edits over suggestions that did require
* edits. This is a known limitation.
*
* <p>
* Note: complex query analyzers can have a significant impact on the lookup
* performance. It's recommended to not use analyzers that drop or inject terms
* like synonyms to keep the complexity of the prefix intersection low for good
* lookup performance. At index time, complex analyzers can safely be used.
* </p>
*/
public final class XFuzzySuggester extends XAnalyzingSuggester {
private final int maxEdits;
private final boolean transpositions;
private final int nonFuzzyPrefix;
private final int minFuzzyLength;
/**
* The default minimum length of the key passed to {@link
* #lookup} before any edits are allowed.
*/
public static final int DEFAULT_MIN_FUZZY_LENGTH = 3;
/**
* The default prefix length where edits are not allowed.
*/
public static final int DEFAULT_NON_FUZZY_PREFIX = 1;
/**
* The default maximum number of edits for fuzzy
* suggestions.
*/
public static final int DEFAULT_MAX_EDITS = 1;
/**
* The default transposition value passed to {@link org.apache.lucene.util.automaton.LevenshteinAutomata}
*/
public static final boolean DEFAULT_TRANSPOSITIONS = true;
/**
* Creates a {@link FuzzySuggester} instance initialized with default values.
*
* @param analyzer the analyzer used for this suggester
*/
public XFuzzySuggester(Analyzer analyzer) {
this(analyzer, analyzer);
}
/**
* Creates a {@link FuzzySuggester} instance with an index & a query analyzer initialized with default values.
*
* @param indexAnalyzer
* Analyzer that will be used for analyzing suggestions while building the index.
* @param queryAnalyzer
* Analyzer that will be used for analyzing query text during lookup
*/
public XFuzzySuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer) {
this(indexAnalyzer, queryAnalyzer, EXACT_FIRST | PRESERVE_SEP, 256, -1, DEFAULT_MAX_EDITS, DEFAULT_TRANSPOSITIONS,
DEFAULT_NON_FUZZY_PREFIX, DEFAULT_MIN_FUZZY_LENGTH, null, false, 0);
}
/**
* Creates a {@link FuzzySuggester} instance.
*
* @param indexAnalyzer Analyzer that will be used for
* analyzing suggestions while building the index.
* @param queryAnalyzer Analyzer that will be used for
* analyzing query text during lookup
* @param options see {@link #EXACT_FIRST}, {@link #PRESERVE_SEP}
* @param maxSurfaceFormsPerAnalyzedForm Maximum number of
* surface forms to keep for a single analyzed form.
* When there are too many surface forms we discard the
* lowest weighted ones.
* @param maxGraphExpansions Maximum number of graph paths
* to expand from the analyzed form. Set this to -1 for
* no limit.
* @param maxEdits must be >= 0 and <= {@link org.apache.lucene.util.automaton.LevenshteinAutomata#MAXIMUM_SUPPORTED_DISTANCE} .
* @param transpositions <code>true</code> if transpositions should be treated as a primitive
* edit operation. If this is false, comparisons will implement the classic
* Levenshtein algorithm.
* @param nonFuzzyPrefix length of common (non-fuzzy) prefix (see default {@link #DEFAULT_NON_FUZZY_PREFIX}
* @param minFuzzyLength minimum length of lookup key before any edits are allowed (see default {@link #DEFAULT_MIN_FUZZY_LENGTH})
*/
public XFuzzySuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer, int options, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions,
int maxEdits, boolean transpositions, int nonFuzzyPrefix, int minFuzzyLength,
FST<PairOutputs.Pair<Long, BytesRef>> fst, boolean hasPayloads, int maxAnalyzedPathsForOneInput) {
super(indexAnalyzer, queryAnalyzer, options, maxSurfaceFormsPerAnalyzedForm, maxGraphExpansions, fst, hasPayloads, maxAnalyzedPathsForOneInput);
if (maxEdits < 0 || maxEdits > LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE) {
throw new IllegalArgumentException("maxEdits must be between 0 and " + LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE);
}
if (nonFuzzyPrefix < 0) {
throw new IllegalArgumentException("nonFuzzyPrefix must not be >= 0 (got " + nonFuzzyPrefix + ")");
}
if (minFuzzyLength < 0) {
throw new IllegalArgumentException("minFuzzyLength must not be >= 0 (got " + minFuzzyLength + ")");
}
this.maxEdits = maxEdits;
this.transpositions = transpositions;
this.nonFuzzyPrefix = nonFuzzyPrefix;
this.minFuzzyLength = minFuzzyLength;
}
@Override
protected List<FSTUtil.Path<PairOutputs.Pair<Long,BytesRef>>> getFullPrefixPaths(List<FSTUtil.Path<PairOutputs.Pair<Long,BytesRef>>> prefixPaths,
Automaton lookupAutomaton,
FST<PairOutputs.Pair<Long,BytesRef>> fst)
throws IOException {
// TODO: right now there's no penalty for fuzzy/edits,
// ie a completion whose prefix matched exactly what the
// user typed gets no boost over completions that
// required an edit, which get no boost over completions
// requiring two edits. I suspect a multiplicative
// factor is appropriate (eg, say a fuzzy match must be at
// least 2X better weight than the non-fuzzy match to
// "compete") ... in which case I think the wFST needs
// to be log weights or something ...
Automaton levA = toLevenshteinAutomata(lookupAutomaton);
/*
Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8");
w.write(levA.toDot());
w.close();
System.out.println("Wrote LevA to out.dot");
*/
return FSTUtil.intersectPrefixPaths(levA, fst);
}
Automaton toLevenshteinAutomata(Automaton automaton) {
final Set<IntsRef> ref = SpecialOperations.getFiniteStrings(automaton, -1);
Automaton subs[] = new Automaton[ref.size()];
int upto = 0;
for (IntsRef path : ref) {
if (path.length <= nonFuzzyPrefix || path.length < minFuzzyLength) {
subs[upto] = BasicAutomata.makeString(path.ints, path.offset, path.length);
upto++;
} else {
Automaton prefix = BasicAutomata.makeString(path.ints, path.offset, nonFuzzyPrefix);
int ints[] = new int[path.length-nonFuzzyPrefix];
System.arraycopy(path.ints, path.offset+nonFuzzyPrefix, ints, 0, ints.length);
// TODO: maybe add alphaMin to LevenshteinAutomata,
// and pass 1 instead of 0? We probably don't want
// to allow the trailing dedup bytes to be
// edited... but then 0 byte is "in general" allowed
// on input (but not in UTF8).
LevenshteinAutomata lev = new LevenshteinAutomata(ints, 255, transpositions);
Automaton levAutomaton = lev.toAutomaton(maxEdits);
Automaton combined = BasicOperations.concatenate(Arrays.asList(prefix, levAutomaton));
combined.setDeterministic(true); // its like the special case in concatenate itself, except we cloneExpanded already
subs[upto] = combined;
upto++;
}
}
if (subs.length == 0) {
// automaton is empty, there is no accepted paths through it
return BasicAutomata.makeEmpty(); // matches nothing
} else if (subs.length == 1) {
// no synonyms or anything: just a single path through the tokenstream
return subs[0];
} else {
// multiple paths: this is really scary! is it slow?
// maybe we should not do this and throw UOE?
Automaton a = BasicOperations.union(Arrays.asList(subs));
// TODO: we could call toLevenshteinAutomata() before det?
// this only happens if you have multiple paths anyway (e.g. synonyms)
BasicOperations.determinize(a);
return a;
}
}
}

32
src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.codecs.*;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.analyzing.XAnalyzingSuggester;
import org.apache.lucene.search.suggest.analyzing.XFuzzySuggester;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
@ -57,7 +58,7 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
private int maxGraphExpansions;
private boolean hasPayloads;
private final XAnalyzingSuggester prototype;
public AnalyzingCompletionLookupProvider(boolean preserveSep, boolean exactFirst, boolean preservePositionIncrements, boolean hasPayloads) {
this.preserveSep = preserveSep;
this.preservePositionIncrements = preservePositionIncrements;
@ -212,19 +213,28 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
}
return new LookupFactory() {
@Override
public Lookup getLookup(FieldMapper<?> mapper, boolean exactFirst) {
public Lookup getLookup(FieldMapper<?> mapper, CompletionSuggestionContext suggestionContext) {
AnalyzingSuggestHolder analyzingSuggestHolder = lookupMap.get(mapper.names().fullName());
if (analyzingSuggestHolder == null) {
return null;
}
int flags = exactFirst ? XAnalyzingSuggester.EXACT_FIRST : 0;
if (analyzingSuggestHolder.preserveSep) {
flags |= XAnalyzingSuggester.PRESERVE_SEP;
int flags = analyzingSuggestHolder.preserveSep? XAnalyzingSuggester.PRESERVE_SEP : 0;
XAnalyzingSuggester suggester;
if (suggestionContext.isFuzzy()) {
suggester = new XFuzzySuggester(mapper.indexAnalyzer(), mapper.searchAnalyzer(), flags,
analyzingSuggestHolder.maxSurfaceFormsPerAnalyzedForm, analyzingSuggestHolder.maxGraphExpansions,
suggestionContext.getFuzzyEditDistance(), suggestionContext.isFuzzyTranspositions(),
suggestionContext.getFuzzyNonPrefixLength(), suggestionContext.getFuzzyMinPrefixLength(),
analyzingSuggestHolder.fst, analyzingSuggestHolder.hasPayloads,
analyzingSuggestHolder.maxAnalyzedPathsForOneInput);
} else {
suggester = new XAnalyzingSuggester(mapper.indexAnalyzer(), mapper.searchAnalyzer(), flags,
analyzingSuggestHolder.maxSurfaceFormsPerAnalyzedForm, analyzingSuggestHolder.maxGraphExpansions,
analyzingSuggestHolder.fst, analyzingSuggestHolder.hasPayloads,
analyzingSuggestHolder.maxAnalyzedPathsForOneInput);
}
XAnalyzingSuggester suggester = new XAnalyzingSuggester(mapper.indexAnalyzer(), mapper.searchAnalyzer(), flags,
analyzingSuggestHolder.maxSurfaceFormsPerAnalyzedForm, analyzingSuggestHolder.maxGraphExpansions,
analyzingSuggestHolder.fst, analyzingSuggestHolder.hasPayloads,
analyzingSuggestHolder.maxAnalyzedPathsForOneInput);
suggester.setPreservePositionIncrements(analyzingSuggestHolder.preservePositionIncrements);
return suggester;
}
@ -240,8 +250,8 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
final int maxAnalyzedPathsForOneInput;
final FST<Pair<Long, BytesRef>> fst;
public AnalyzingSuggestHolder(boolean preserveSep, boolean preservePositionIncrements, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions, boolean hasPayloads,
int maxAnalyzedPathsForOneInput, FST<Pair<Long, BytesRef>> fst) {
public AnalyzingSuggestHolder(boolean preserveSep, boolean preservePositionIncrements, int maxSurfaceFormsPerAnalyzedForm, int maxGraphExpansions,
boolean hasPayloads, int maxAnalyzedPathsForOneInput, FST<Pair<Long, BytesRef>> fst) {
this.preserveSep = preserveSep;
this.preservePositionIncrements = preservePositionIncrements;
this.maxSurfaceFormsPerAnalyzedForm = maxSurfaceFormsPerAnalyzedForm;

6
src/main/java/org/elasticsearch/search/suggest/completion/Completion090PostingsFormat.java
View File

@ -271,8 +271,8 @@ public class Completion090PostingsFormat extends PostingsFormat {
this.lookup = lookup;
}
public Lookup getLookup(FieldMapper<?> mapper, boolean exactFirst) {
return lookup.getLookup(mapper, exactFirst);
public Lookup getLookup(FieldMapper<?> mapper, CompletionSuggestionContext suggestionContext) {
return lookup.getLookup(mapper, suggestionContext);
}
}
@ -327,6 +327,6 @@ public class Completion090PostingsFormat extends PostingsFormat {
}
public static abstract class LookupFactory {
public abstract Lookup getLookup(FieldMapper<?> mapper, boolean exactFirst);
public abstract Lookup getLookup(FieldMapper<?> mapper, CompletionSuggestionContext suggestionContext);
}
}

27
src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java
View File

@ -48,12 +48,35 @@ public class CompletionSuggestParser implements SuggestContextParser {
if (token == XContentParser.Token.FIELD_NAME) {
fieldName = parser.currentName();
} else if (token.isValue()) {
parseSuggestContext(parser, mapperService, fieldName, suggestion);
suggestion.mapper(mapperService.smartNameFieldMapper(suggestion.getField()));
if (!parseSuggestContext(parser, mapperService, fieldName, suggestion)) {
if (token == XContentParser.Token.VALUE_BOOLEAN && "fuzzy".equals(fieldName)) {
suggestion.setFuzzy(parser.booleanValue());
}
}
} else if (token == XContentParser.Token.START_OBJECT && "fuzzy".equals(fieldName)) {
suggestion.setFuzzy(true);
String fuzzyConfigName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
fuzzyConfigName = parser.currentName();
} else if (token.isValue()) {
if ("edit_distance".equals(fuzzyConfigName) || "editDistance".equals(fuzzyConfigName)) {
suggestion.setFuzzyEditDistance(parser.intValue());
} else if ("transpositions".equals(fuzzyConfigName)) {
suggestion.setFuzzyTranspositions(parser.booleanValue());
} else if ("min_prefix_len".equals(fuzzyConfigName) || "minPrefixLen".equals(fuzzyConfigName)) {
suggestion.setFuzzyMinPrefixLength(parser.intValue());
} else if ("non_prefix_len".equals(fuzzyConfigName) || "non_prefix_len".equals(fuzzyConfigName)) {
suggestion.setFuzzyNonPrefixLength(parser.intValue());
}
}
}
} else {
throw new ElasticSearchIllegalArgumentException("suggester[completion] doesn't support field [" + fieldName + "]");
}
}
suggestion.mapper(mapperService.smartNameFieldMapper(suggestion.getField()));
return suggestion;
}

3
src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
View File

@ -48,6 +48,7 @@ public class CompletionSuggester implements Suggester<CompletionSuggestionContex
@Override
public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> execute(String name,
CompletionSuggestionContext suggestionContext, IndexReader indexReader, CharsRef spare) throws IOException {
CompletionSuggestion completionSuggestionSuggestion = new CompletionSuggestion(name, suggestionContext.getSize());
CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(new StringText(suggestionContext.getText()
.utf8ToString()), 0, suggestionContext.getText().toString().length());
@ -65,7 +66,7 @@ public class CompletionSuggester implements Suggester<CompletionSuggestionContex
Terms terms = atomicReader.fields().terms(fieldName);
if (terms instanceof Completion090PostingsFormat.CompletionTerms) {
Completion090PostingsFormat.CompletionTerms lookupTerms = (Completion090PostingsFormat.CompletionTerms) terms;
Lookup lookup = lookupTerms.getLookup(suggestionContext.mapper(), false);
Lookup lookup = lookupTerms.getLookup(suggestionContext.mapper(), suggestionContext);
List<Lookup.LookupResult> lookupResults = lookup.lookup(prefix, false, suggestionContext.getSize());
for (Lookup.LookupResult res : lookupResults) {

4
src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.search.suggest.completion;
import org.apache.lucene.search.suggest.analyzing.XFuzzySuggester;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.search.suggest.SuggestBuilder;
@ -34,8 +35,7 @@ public class CompletionSuggestionBuilder extends SuggestBuilder.SuggestionBuilde
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
return builder;
}
}

46
src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.search.suggest.completion;
import org.apache.lucene.search.suggest.analyzing.XFuzzySuggester;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionSearchContext;
@ -28,6 +29,11 @@ import org.elasticsearch.search.suggest.SuggestionSearchContext;
public class CompletionSuggestionContext extends SuggestionSearchContext.SuggestionContext {
private FieldMapper<?> mapper;
private int fuzzyEditDistance = XFuzzySuggester.DEFAULT_MAX_EDITS;
private boolean fuzzyTranspositions = XFuzzySuggester.DEFAULT_TRANSPOSITIONS;
private int fuzzyMinPrefixLength = XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH;
private int fuzzyNonPrefixLength = XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX;
private boolean fuzzy = false;
public CompletionSuggestionContext(Suggester suggester) {
super(suggester);
@ -40,4 +46,44 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest
public void mapper(FieldMapper<?> mapper) {
this.mapper = mapper;
}
public void setFuzzyEditDistance(int fuzzyEditDistance) {
this.fuzzyEditDistance = fuzzyEditDistance;
}
public int getFuzzyEditDistance() {
return fuzzyEditDistance;
}
public void setFuzzyTranspositions(boolean fuzzyTranspositions) {
this.fuzzyTranspositions = fuzzyTranspositions;
}
public boolean isFuzzyTranspositions() {
return fuzzyTranspositions;
}
public void setFuzzyMinPrefixLength(int fuzzyMinPrefixLength) {
this.fuzzyMinPrefixLength = fuzzyMinPrefixLength;
}
public int getFuzzyMinPrefixLength() {
return fuzzyMinPrefixLength;
}
public void setFuzzyNonPrefixLength(int fuzzyNonPrefixLength) {
this.fuzzyNonPrefixLength = fuzzyNonPrefixLength;
}
public int getFuzzyNonPrefixLength() {
return fuzzyNonPrefixLength;
}
public void setFuzzy(boolean fuzzy) {
this.fuzzy = fuzzy;
}
public boolean isFuzzy() {
return fuzzy;
}
}

98
src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionFuzzyBuilder.java Normal file
View File

@ -0,0 +1,98 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.suggest.completion;
import org.apache.lucene.search.suggest.analyzing.XFuzzySuggester;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.search.suggest.SuggestBuilder;
import java.io.IOException;
/**
*
*/
public class CompletionSuggestionFuzzyBuilder extends SuggestBuilder.SuggestionBuilder<CompletionSuggestionFuzzyBuilder> {
public CompletionSuggestionFuzzyBuilder(String name) {
super(name, "completion");
}
private int fuzzyEditDistance = XFuzzySuggester.DEFAULT_MAX_EDITS;
private boolean fuzzyTranspositions = XFuzzySuggester.DEFAULT_TRANSPOSITIONS;
private int fuzzyMinPrefixLength = XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH;
private int fuzzyNonPrefixLength = XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX;
public int getFuzzyEditDistance() {
return fuzzyEditDistance;
}
public CompletionSuggestionFuzzyBuilder setFuzzyEditDistance(int fuzzyEditDistance) {
this.fuzzyEditDistance = fuzzyEditDistance;
return this;
}
public boolean isFuzzyTranspositions() {
return fuzzyTranspositions;
}
public CompletionSuggestionFuzzyBuilder setFuzzyTranspositions(boolean fuzzyTranspositions) {
this.fuzzyTranspositions = fuzzyTranspositions;
return this;
}
public int getFuzzyMinPrefixLength() {
return fuzzyMinPrefixLength;
}
public CompletionSuggestionFuzzyBuilder setFuzzyMinPrefixLength(int fuzzyMinPrefixLength) {
this.fuzzyMinPrefixLength = fuzzyMinPrefixLength;
return this;
}
public int getFuzzyNonPrefixLength() {
return fuzzyNonPrefixLength;
}
public CompletionSuggestionFuzzyBuilder setFuzzyNonPrefixLength(int fuzzyNonPrefixLength) {
this.fuzzyNonPrefixLength = fuzzyNonPrefixLength;
return this;
}
@Override
protected XContentBuilder innerToXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject("fuzzy");
if (fuzzyEditDistance != XFuzzySuggester.DEFAULT_MAX_EDITS) {
builder.field("edit_distance", fuzzyEditDistance);
}
if (fuzzyTranspositions != XFuzzySuggester.DEFAULT_TRANSPOSITIONS) {
builder.field("transpositions", fuzzyTranspositions);
}
if (fuzzyMinPrefixLength != XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH) {
builder.field("min_prefix_len", fuzzyMinPrefixLength);
}
if (fuzzyNonPrefixLength != XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX) {
builder.field("non_prefix_len", fuzzyNonPrefixLength);
}
builder.endObject();
return builder;
}
}

5
src/test/java/org/elasticsearch/test/integration/search/suggest/CompletionPostingsFormatTest.java
View File

@ -47,6 +47,7 @@ import org.elasticsearch.search.suggest.SuggestUtils;
import org.elasticsearch.search.suggest.completion.AnalyzingCompletionLookupProvider;
import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat;
import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat.LookupFactory;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionContext;
import org.elasticsearch.test.integration.ElasticsearchTestCase;
import org.junit.Test;
@ -90,7 +91,7 @@ public class CompletionPostingsFormatTest extends ElasticsearchTestCase {
LookupFactory load = provider.load(input);
PostingsFormatProvider format = new PreBuiltPostingsFormatProvider(new ElasticSearch090PostingsFormat());
NamedAnalyzer analyzer = new NamedAnalyzer("foo", new StandardAnalyzer(TEST_VERSION_CURRENT));
Lookup lookup = load.getLookup(new CompletionFieldMapper(new Names("foo"), analyzer, analyzer, format, null, true, true, true), false);
Lookup lookup = load.getLookup(new CompletionFieldMapper(new Names("foo"), analyzer, analyzer, format, null, true, true, true), new CompletionSuggestionContext(null));
List<LookupResult> result = lookup.lookup("ge", false, 10);
assertThat(result.get(0).key.toString(), equalTo("Generator - Foo Fighters"));
assertThat(result.get(0).payload.utf8ToString(), equalTo("id:10"));
@ -240,7 +241,7 @@ public class CompletionPostingsFormatTest extends ElasticsearchTestCase {
assertThat(reader.leaves().get(0).reader().numDocs(), equalTo(weights.length));
AtomicReaderContext atomicReaderContext = reader.leaves().get(0);
Terms luceneTerms = atomicReaderContext.reader().terms(mapper.name());
Lookup lookup = ((Completion090PostingsFormat.CompletionTerms) luceneTerms).getLookup(mapper, false);
Lookup lookup = ((Completion090PostingsFormat.CompletionTerms) luceneTerms).getLookup(mapper, new CompletionSuggestionContext(null));
reader.close();
writer.close();
dir.close();

119
src/test/java/org/elasticsearch/test/integration/search/suggest/CompletionSuggestSearchTests.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.index.mapper.MapperException;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionFuzzyBuilder;
import org.elasticsearch.test.integration.AbstractSharedClusterTest;
import org.junit.Test;
@ -333,6 +334,124 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters");
}
@Test
public void testThatFuzzySuggesterWorks() throws Exception {
createIndexAndMapping("simple", "simple", true, true, true);
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD)
.startArray("input").value("Nirvana").endArray()
.endObject().endObject()
).get();
refresh();
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nirv").size(10)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo", "Nirvana");
suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nirw").size(10)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo", "Nirvana");
}
@Test
public void testThatFuzzySuggesterSupportsEditDistances() throws Exception {
createIndexAndMapping("simple", "simple", true, true, true);
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD)
.startArray("input").value("Nirvana").endArray()
.endObject().endObject()
).get();
refresh();
// edit distance 1
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Norw").size(10)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo");
// edit distance 2
suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Norw").size(10).setFuzzyEditDistance(2)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo", "Nirvana");
}
@Test
public void testThatFuzzySuggesterSupportsTranspositions() throws Exception {
createIndexAndMapping("simple", "simple", true, true, true);
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD)
.startArray("input").value("Nirvana").endArray()
.endObject().endObject()
).get();
refresh();
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nriv").size(10).setFuzzyTranspositions(false).setFuzzyEditDistance(1)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo");
suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nriv").size(10).setFuzzyTranspositions(true).setFuzzyEditDistance(1)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo", "Nirvana");
}
@Test
public void testThatFuzzySuggesterSupportsMinPrefixLength() throws Exception {
createIndexAndMapping("simple", "simple", true, true, true);
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD)
.startArray("input").value("Nirvana").endArray()
.endObject().endObject()
).get();
refresh();
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nriva").size(10).setFuzzyMinPrefixLength(6)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo");
suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nrivan").size(10).setFuzzyMinPrefixLength(6)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo", "Nirvana");
}
@Test
public void testThatFuzzySuggesterSupportsNonPrefixLength() throws Exception {
createIndexAndMapping("simple", "simple", true, true, true);
client().prepareIndex(INDEX, TYPE, "1").setSource(jsonBuilder()
.startObject().startObject(FIELD)
.startArray("input").value("Nirvana").endArray()
.endObject().endObject()
).get();
refresh();
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nirw").size(10).setFuzzyNonPrefixLength(4)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo");
suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nirvo").size(10).setFuzzyNonPrefixLength(4)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo", "Nirvana");
}
public void assertSuggestions(String suggestion, String ... suggestions) {
String suggestionName = RandomStrings.randomAsciiOfLength(new Random(), 10);
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(