honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-28 07:59:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

CompletionSuggester cleanups

Browse Source 
* Fuzzy Suggester parameter names are now easier to understand
  * non_prefix_length became prefix_length
  * min_prefix_length became min_length
* Instead of specyfying search_analyzer and index_analyzer using analyzer for both is supported
* CompletionSuggester used the CharsRef spare instead of too much toString() now
This commit is contained in:
Alexander Reelsen 2013-08-15 13:24:27 +02:00
parent bfac2f575e
commit f644ae5550
8 changed files with 86 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
View File

@ -65,6 +65,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
}
public static class Fields {
public static final String ANALYZER = "analyzer";
public static final String INDEX_ANALYZER = "index_analyzer";
public static final String SEARCH_ANALYZER = "search_analyzer";
public static final String PRESERVE_SEPARATORS = "preserve_separators";
@ -127,7 +128,10 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
if (fieldName.equals("type")) {
continue;
}
if (fieldName.equals(Fields.INDEX_ANALYZER) || fieldName.equals("indexAnalyzer")) {
if (fieldName.equals("analyzer")) {
builder.indexAnalyzer(parserContext.analysisService().analyzer(fieldNode.toString()));
builder.searchAnalyzer(parserContext.analysisService().analyzer(fieldNode.toString()));
} else if (fieldName.equals(Fields.INDEX_ANALYZER) || fieldName.equals("indexAnalyzer")) {
builder.indexAnalyzer(parserContext.analysisService().analyzer(fieldNode.toString()));
} else if (fieldName.equals(Fields.SEARCH_ANALYZER) || fieldName.equals("searchAnalyzer")) {
builder.searchAnalyzer(parserContext.analysisService().analyzer(fieldNode.toString()));
@ -238,7 +242,7 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
surfaceForm), weight, payload);
for (String input : inputs) {
context.doc().add(getCompletionField(input, suggestPayload));
}
}
}
}
@ -271,14 +275,20 @@ public class CompletionFieldMapper extends AbstractFieldMapper<String> {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.startObject(name())
.field(Fields.TYPE, CONTENT_TYPE)
.field(Fields.INDEX_ANALYZER, indexAnalyzer.name())
.field(Fields.SEARCH_ANALYZER, searchAnalyzer.name())
.field(Fields.PAYLOADS, this.payloads)
builder.startObject(name())
.field(Fields.TYPE, CONTENT_TYPE);
if (indexAnalyzer.name().equals(searchAnalyzer.name())) {
builder.field(Fields.ANALYZER, indexAnalyzer.name());
} else {
builder.field(Fields.INDEX_ANALYZER, indexAnalyzer.name())
.field(Fields.SEARCH_ANALYZER, searchAnalyzer.name());
}
builder.field(Fields.PAYLOADS, this.payloads)
.field(Fields.PRESERVE_SEPARATORS, this.preserveSeparators)
.field(Fields.PRESERVE_POSITION_INCREMENTS, this.preservePositionIncrements)
.endObject();
return builder;
}
@Override

2
src/main/java/org/elasticsearch/search/suggest/completion/AnalyzingCompletionLookupProvider.java
View File

@ -225,7 +225,7 @@ public class AnalyzingCompletionLookupProvider extends CompletionLookupProvider
suggester = new XFuzzySuggester(mapper.indexAnalyzer(), mapper.searchAnalyzer(), flags,
analyzingSuggestHolder.maxSurfaceFormsPerAnalyzedForm, analyzingSuggestHolder.maxGraphExpansions,
suggestionContext.getFuzzyEditDistance(), suggestionContext.isFuzzyTranspositions(),
suggestionContext.getFuzzyNonPrefixLength(), suggestionContext.getFuzzyMinPrefixLength(),
suggestionContext.getFuzzyPrefixLength(), suggestionContext.getFuzzyMinLength(),
analyzingSuggestHolder.fst, analyzingSuggestHolder.hasPayloads,
analyzingSuggestHolder.maxAnalyzedPathsForOneInput);

8
src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java
View File

@ -64,10 +64,10 @@ public class CompletionSuggestParser implements SuggestContextParser {
suggestion.setFuzzyEditDistance(parser.intValue());
} else if ("transpositions".equals(fuzzyConfigName)) {
suggestion.setFuzzyTranspositions(parser.booleanValue());
} else if ("min_prefix_len".equals(fuzzyConfigName) || "minPrefixLen".equals(fuzzyConfigName)) {
suggestion.setFuzzyMinPrefixLength(parser.intValue());
} else if ("non_prefix_len".equals(fuzzyConfigName) || "non_prefix_len".equals(fuzzyConfigName)) {
suggestion.setFuzzyNonPrefixLength(parser.intValue());
} else if ("min_length".equals(fuzzyConfigName) || "minLength".equals(fuzzyConfigName)) {
suggestion.setFuzzyMinLength(parser.intValue());
} else if ("prefix_length".equals(fuzzyConfigName) || "prefixLength".equals(fuzzyConfigName)) {
suggestion.setFuzzyPrefixLength(parser.intValue());
}
}
}

22
src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.UnicodeUtil;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.text.StringText;
@ -49,17 +50,17 @@ public class CompletionSuggester implements Suggester<CompletionSuggestionContex
public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> execute(String name,
CompletionSuggestionContext suggestionContext, IndexReader indexReader, CharsRef spare) throws IOException {
CompletionSuggestion completionSuggestionSuggestion = new CompletionSuggestion(name, suggestionContext.getSize());
CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(new StringText(suggestionContext.getText()
.utf8ToString()), 0, suggestionContext.getText().toString().length());
completionSuggestionSuggestion.addTerm(completionSuggestEntry);
String fieldName = suggestionContext.getField();
if (suggestionContext.mapper() == null || !(suggestionContext.mapper() instanceof CompletionFieldMapper)) {
throw new ElasticSearchException("Field [" + suggestionContext.getField() + "] is not a completion suggest field");
}
String prefix = suggestionContext.getText().utf8ToString();
CompletionSuggestion completionSuggestion = new CompletionSuggestion(name, suggestionContext.getSize());
UnicodeUtil.UTF8toUTF16(suggestionContext.getText(), spare);
CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(new StringText(spare.toString()), 0, spare.length());
completionSuggestion.addTerm(completionSuggestEntry);
String fieldName = suggestionContext.getField();
Map<String, CompletionSuggestion.Entry.Option> results = Maps.newHashMapWithExpectedSize(indexReader.leaves().size() * suggestionContext.getSize());
for (AtomicReaderContext atomicReaderContext : indexReader.leaves()) {
AtomicReader atomicReader = atomicReaderContext.reader();
@ -67,7 +68,7 @@ public class CompletionSuggester implements Suggester<CompletionSuggestionContex
if (terms instanceof Completion090PostingsFormat.CompletionTerms) {
Completion090PostingsFormat.CompletionTerms lookupTerms = (Completion090PostingsFormat.CompletionTerms) terms;
Lookup lookup = lookupTerms.getLookup(suggestionContext.mapper(), suggestionContext);
List<Lookup.LookupResult> lookupResults = lookup.lookup(prefix, false, suggestionContext.getSize());
List<Lookup.LookupResult> lookupResults = lookup.lookup(spare, false, suggestionContext.getSize());
for (Lookup.LookupResult res : lookupResults) {
final String key = res.key.toString();
@ -87,11 +88,12 @@ public class CompletionSuggester implements Suggester<CompletionSuggestionContex
final List<CompletionSuggestion.Entry.Option> options = new ArrayList<CompletionSuggestion.Entry.Option>(results.values());
CollectionUtil.introSort(options, scoreComparator);
for (int i = 0 ; i < Math.min(suggestionContext.getSize(), options.size()) ; i++) {
int optionCount = Math.min(suggestionContext.getSize(), options.size());
for (int i = 0 ; i < optionCount ; i++) {
completionSuggestEntry.addOption(options.get(i));
}
return completionSuggestionSuggestion;
return completionSuggestion;
}
@Override

20
src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java
View File

@ -31,8 +31,8 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest
private FieldMapper<?> mapper;
private int fuzzyEditDistance = XFuzzySuggester.DEFAULT_MAX_EDITS;
private boolean fuzzyTranspositions = XFuzzySuggester.DEFAULT_TRANSPOSITIONS;
private int fuzzyMinPrefixLength = XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH;
private int fuzzyNonPrefixLength = XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX;
private int fuzzyMinLength = XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH;
private int fuzzyPrefixLength = XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX;
private boolean fuzzy = false;
public CompletionSuggestionContext(Suggester suggester) {
@ -63,20 +63,20 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest
return fuzzyTranspositions;
}
public void setFuzzyMinPrefixLength(int fuzzyMinPrefixLength) {
this.fuzzyMinPrefixLength = fuzzyMinPrefixLength;
public void setFuzzyMinLength(int fuzzyMinPrefixLength) {
this.fuzzyMinLength = fuzzyMinPrefixLength;
}
public int getFuzzyMinPrefixLength() {
return fuzzyMinPrefixLength;
public int getFuzzyMinLength() {
return fuzzyMinLength;
}
public void setFuzzyNonPrefixLength(int fuzzyNonPrefixLength) {
this.fuzzyNonPrefixLength = fuzzyNonPrefixLength;
public void setFuzzyPrefixLength(int fuzzyNonPrefixLength) {
this.fuzzyPrefixLength = fuzzyNonPrefixLength;
}
public int getFuzzyNonPrefixLength() {
return fuzzyNonPrefixLength;
public int getFuzzyPrefixLength() {
return fuzzyPrefixLength;
}
public void setFuzzy(boolean fuzzy) {

28
src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionFuzzyBuilder.java
View File

@ -36,8 +36,8 @@ public class CompletionSuggestionFuzzyBuilder extends SuggestBuilder.SuggestionB
private int fuzzyEditDistance = XFuzzySuggester.DEFAULT_MAX_EDITS;
private boolean fuzzyTranspositions = XFuzzySuggester.DEFAULT_TRANSPOSITIONS;
private int fuzzyMinPrefixLength = XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH;
private int fuzzyNonPrefixLength = XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX;
private int fuzzyMinLength = XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH;
private int fuzzyPrefixLength = XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX;
public int getFuzzyEditDistance() {
return fuzzyEditDistance;
@ -57,21 +57,21 @@ public class CompletionSuggestionFuzzyBuilder extends SuggestBuilder.SuggestionB
return this;
}
public int getFuzzyMinPrefixLength() {
return fuzzyMinPrefixLength;
public int getFuzzyMinLength() {
return fuzzyMinLength;
}
public CompletionSuggestionFuzzyBuilder setFuzzyMinPrefixLength(int fuzzyMinPrefixLength) {
this.fuzzyMinPrefixLength = fuzzyMinPrefixLength;
public CompletionSuggestionFuzzyBuilder setFuzzyMinLength(int fuzzyMinLength) {
this.fuzzyMinLength = fuzzyMinLength;
return this;
}
public int getFuzzyNonPrefixLength() {
return fuzzyNonPrefixLength;
public int getFuzzyPrefixLength() {
return fuzzyPrefixLength;
}
public CompletionSuggestionFuzzyBuilder setFuzzyNonPrefixLength(int fuzzyNonPrefixLength) {
this.fuzzyNonPrefixLength = fuzzyNonPrefixLength;
public CompletionSuggestionFuzzyBuilder setFuzzyPrefixLength(int fuzzyPrefixLength) {
this.fuzzyPrefixLength = fuzzyPrefixLength;
return this;
}
@ -85,11 +85,11 @@ public class CompletionSuggestionFuzzyBuilder extends SuggestBuilder.SuggestionB
if (fuzzyTranspositions != XFuzzySuggester.DEFAULT_TRANSPOSITIONS) {
builder.field("transpositions", fuzzyTranspositions);
}
if (fuzzyMinPrefixLength != XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH) {
builder.field("min_prefix_len", fuzzyMinPrefixLength);
if (fuzzyMinLength != XFuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH) {
builder.field("min_length", fuzzyMinLength);
}
if (fuzzyNonPrefixLength != XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX) {
builder.field("non_prefix_len", fuzzyNonPrefixLength);
if (fuzzyPrefixLength != XFuzzySuggester.DEFAULT_NON_FUZZY_PREFIX) {
builder.field("prefix_length", fuzzyPrefixLength);
}
builder.endObject();

8
src/test/java/org/elasticsearch/test/integration/search/suggest/CompletionSuggestSearchTests.java
View File

@ -418,12 +418,12 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
refresh();
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nriva").size(10).setFuzzyMinPrefixLength(6)
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nriva").size(10).setFuzzyMinLength(6)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo");
suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nrivan").size(10).setFuzzyMinPrefixLength(6)
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nrivan").size(10).setFuzzyMinLength(6)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo", "Nirvana");
}
@ -441,12 +441,12 @@ public class CompletionSuggestSearchTests extends AbstractSharedClusterTest {
refresh();
SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nirw").size(10).setFuzzyNonPrefixLength(4)
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nirw").size(10).setFuzzyPrefixLength(4)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo");
suggestResponse = client().prepareSuggest(INDEX).addSuggestion(
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nirvo").size(10).setFuzzyNonPrefixLength(4)
new CompletionSuggestionFuzzyBuilder("foo").field(FIELD).text("Nirvo").size(10).setFuzzyPrefixLength(4)
).execute().actionGet();
assertSuggestions(suggestResponse, false, "foo", "Nirvana");
}

24
src/test/java/org/elasticsearch/test/unit/index/mapper/completion/CompletionFieldMapperTests.java
View File

@ -85,4 +85,28 @@ public class CompletionFieldMapperTests {
assertThat(Boolean.valueOf(configMap.get("preserve_position_increments").toString()), is(true));
}
@Test
public void testThatSerializationCombinesToOneAnalyzerFieldIfBothAreEqual() throws Exception {
String mapping = jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("completion")
.field("type", "completion")
.field("index_analyzer", "simple")
.field("search_analyzer", "simple")
.endObject().endObject()
.endObject().endObject().string();
DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping);
FieldMapper fieldMapper = defaultMapper.mappers().name("completion").mapper();
assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class));
CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
XContentBuilder builder = jsonBuilder().startObject();
completionFieldMapper.toXContent(builder, null).endObject();
builder.close();
Map<String, Object> serializedMap = JsonXContent.jsonXContent.createParser(builder.bytes()).mapAndClose();
Map<String, Object> configMap = (Map<String, Object>) serializedMap.get("completion");
assertThat(configMap.get("analyzer").toString(), is("simple"));
}
}