honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix suggestions for empty indices (#42927) 

Currently suggesters return null values on empty shards. Usually this gets replaced
by results from other non-epmty shards, but if the index is completely epmty (e.g. after
creation) the search responses "suggest" is also "null" and we don't render a corresponding
output in the REST response. This is an irritating edge case that requires special handling on
the user side (see #42473) and should be fixed.

This change makes sure every suggester type (completion, terms, phrase) returns at least an
empty skeleton suggestion output, even for empty shards. This way, even if we don't find
any suggestions anywhere, we still return and output the empty suggestion.

Closes #42473
This commit is contained in:
Christoph Büscher 2019-06-12 15:41:41 +02:00
parent 5ae2460782
commit 7f690e8606
9 changed files with 151 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.common.text.Text;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.Suggester;
import java.io.IOException;
import java.util.Locale;
public class CustomSuggester extends Suggester<CustomSuggestionContext> {
@ -35,15 +36,12 @@ public class CustomSuggester extends Suggester<CustomSuggestionContext> {
String name,
CustomSuggestionContext suggestion,
IndexSearcher searcher,
CharsRefBuilder spare) {
// Get the suggestion context
String text = suggestion.getText().utf8ToString();
CharsRefBuilder spare) throws IOException {
// create two suggestions with 12 and 123 appended
CustomSuggestion response = new CustomSuggestion(name, suggestion.getSize(), "suggestion-dummy-value");
CustomSuggestion.Entry entry = new CustomSuggestion.Entry(new Text(text), 0, text.length(), "entry-dummy-value");
CustomSuggestion response = emptySuggestion(name, suggestion, spare);
CustomSuggestion.Entry entry = response.getEntries().get(0);
String text = entry.getText().string();
String firstOption =
String.format(Locale.ROOT, "%s-%s-%s-%s", text, suggestion.getField(), suggestion.options.get("suffix"), "12");
@ -55,8 +53,16 @@ public class CustomSuggester extends Suggester<CustomSuggestionContext> {
CustomSuggestion.Entry.Option option123 = new CustomSuggestion.Entry.Option(new Text(secondOption), 0.8f, "option-dummy-value-2");
entry.addOption(option123);
response.addTerm(entry);
return response;
}
@Override
protected CustomSuggestion emptySuggestion(String name, CustomSuggestionContext suggestion,
CharsRefBuilder spare) throws IOException {
String text = suggestion.getText().utf8ToString();
CustomSuggestion response = new CustomSuggestion(name, suggestion.getSize(), "suggestion-dummy-value");
CustomSuggestion.Entry entry = new CustomSuggestion.Entry(new Text(text), 0, text.length(), "entry-dummy-value");
response.addTerm(entry);
return response;
}
}

1
server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.action.search;
import com.carrotsearch.hppc.IntArrayList;
import com.carrotsearch.hppc.ObjectObjectHashMap;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.FieldDoc;

7
server/src/main/java/org/elasticsearch/search/suggest/Suggester.java
View File

@ -29,12 +29,15 @@ public abstract class Suggester<T extends SuggestionSearchContext.SuggestionCont
protected abstract Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
innerExecute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException;
protected abstract Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
emptySuggestion(String name, T suggestion, CharsRefBuilder spare) throws IOException;
public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
execute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
// #3469 We want to ignore empty shards
// we only want to output an empty suggestion on empty shards
if (searcher.getIndexReader().numDocs() == 0) {
return null;
return emptySuggestion(name, suggestion, spare);
}
return innerExecute(name, suggestion, searcher, spare);
}

19
server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
View File

@ -49,12 +49,7 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
final CompletionSuggestionContext suggestionContext, final IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
if (suggestionContext.getFieldType() != null) {
final CompletionFieldMapper.CompletionFieldType fieldType = suggestionContext.getFieldType();
CompletionSuggestion completionSuggestion =
new CompletionSuggestion(name, suggestionContext.getSize(), suggestionContext.isSkipDuplicates());
spare.copyUTF8Bytes(suggestionContext.getText());
CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(
new Text(spare.toString()), 0, spare.length());
completionSuggestion.addTerm(completionSuggestEntry);
CompletionSuggestion completionSuggestion = emptySuggestion(name, suggestionContext, spare);
int shardSize = suggestionContext.getShardSize() != null ? suggestionContext.getShardSize() : suggestionContext.getSize();
TopSuggestGroupDocsCollector collector = new TopSuggestGroupDocsCollector(shardSize, suggestionContext.isSkipDuplicates());
suggest(searcher, suggestionContext.toQuery(), collector);
@ -71,7 +66,7 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
if (numResult++ < suggestionContext.getSize()) {
CompletionSuggestion.Entry.Option option = new CompletionSuggestion.Entry.Option(suggestDoc.doc,
new Text(suggestDoc.key.toString()), suggestDoc.score, contexts);
completionSuggestEntry.addOption(option);
completionSuggestion.getEntries().get(0).addOption(option);
} else {
break;
}
@ -96,4 +91,14 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
}
}
}
@Override
protected CompletionSuggestion emptySuggestion(String name, CompletionSuggestionContext suggestion, CharsRefBuilder spare)
throws IOException {
CompletionSuggestion completionSuggestion = new CompletionSuggestion(name, suggestion.getSize(), suggestion.isSkipDuplicates());
spare.copyUTF8Bytes(suggestion.getText());
CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(new Text(spare.toString()), 0, spare.length());
completionSuggestion.addTerm(completionSuggestEntry);
return completionSuggestion;
}
}

9
server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
View File

@ -156,4 +156,13 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
spare.copyUTF8Bytes(suggestion.getText());
return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore);
}
@Override
protected Suggestion<? extends Entry<? extends Option>> emptySuggestion(String name, PhraseSuggestionContext suggestion,
CharsRefBuilder spare) throws IOException {
PhraseSuggestion phraseSuggestion = new PhraseSuggestion(name, suggestion.getSize());
spare.copyUTF8Bytes(suggestion.getText());
phraseSuggestion.addTerm(new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length()));
return phraseSuggestion;
}
}

12
server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
View File

@ -94,4 +94,16 @@ public final class TermSuggester extends Suggester<TermSuggestionContext> {
}
}
@Override
protected TermSuggestion emptySuggestion(String name, TermSuggestionContext suggestion, CharsRefBuilder spare) throws IOException {
TermSuggestion termSuggestion = new TermSuggestion(name, suggestion.getSize(), suggestion.getDirectSpellCheckerSettings().sort());
List<Token> tokens = queryTerms(suggestion, spare);
for (Token token : tokens) {
Text key = new Text(new BytesArray(token.term.bytes()));
TermSuggestion.Entry resultEntry = new TermSuggestion.Entry(key, token.startOffset, token.endOffset - token.startOffset);
termSuggestion.addTerm(resultEntry);
}
return termSuggestion;
}
}

11
server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
View File

@ -66,9 +66,12 @@ import org.elasticsearch.search.rescore.QueryRescorerBuilder;
import org.elasticsearch.search.rescore.RescoreContext;
import org.elasticsearch.search.rescore.RescorerBuilder;
import org.elasticsearch.search.suggest.Suggest.Suggestion;
import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
import org.elasticsearch.search.suggest.Suggester;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.SuggestionSearchContext;
import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
import org.elasticsearch.search.suggest.term.TermSuggestion;
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
import org.elasticsearch.test.ESTestCase;
@ -173,6 +176,7 @@ public class SearchModuleTests extends ESTestCase {
expectThrows(IllegalArgumentException.class, registryForPlugin(registersDupePipelineAggregation));
SearchPlugin registersDupeRescorer = new SearchPlugin() {
@Override
public List<RescorerSpec<?>> getRescorers() {
return singletonList(
new RescorerSpec<>(QueryRescorerBuilder.NAME, QueryRescorerBuilder::new, QueryRescorerBuilder::fromXContent));
@ -525,6 +529,7 @@ public class SearchModuleTests extends ESTestCase {
}
private static class TestSuggester extends Suggester<SuggestionSearchContext.SuggestionContext> {
@Override
protected Suggestion<? extends Suggestion.Entry<? extends Suggestion.Entry.Option>> innerExecute(
String name,
@ -533,6 +538,12 @@ public class SearchModuleTests extends ESTestCase {
CharsRefBuilder spare) throws IOException {
return null;
}
@Override
protected Suggestion<? extends Entry<? extends Option>> emptySuggestion(String name, SuggestionContext suggestion,
CharsRefBuilder spare) throws IOException {
return null;
}
}
private static class TestSuggestionBuilder extends SuggestionBuilder<TestSuggestionBuilder> {

20
server/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
View File

@ -357,6 +357,26 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
}
}
/**
* Suggestions run on an empty index should return a suggest element as part of the response. See #42473 for details.
*/
public void testSuggestEmptyIndex() throws IOException, InterruptedException {
final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
createIndexAndMapping(mapping);
CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("v");
SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
.setFetchSource("a", "b").get();
Suggest suggest = searchResponse.getSuggest();
assertNotNull(suggest);
CompletionSuggestion completionSuggestion = suggest.getSuggestion("foo");
CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0);
assertEquals("v", options.getText().string());
assertEquals(1, options.getLength());
assertEquals(0, options.getOffset());
assertEquals(0, options.options.size());
}
public void testThatWeightsAreWorking() throws Exception {
createIndexAndMapping(completionMappingBuilder);

74
server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
View File

@ -345,6 +345,34 @@ public class SuggestSearchIT extends ESIntegTestCase {
assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
}
public void testEmptyIndex() throws Exception {
assertAcked(prepareCreate("test").addMapping("type1", "text", "type=text"));
ensureGreen();
// use SuggestMode.ALWAYS, otherwise the results can vary between requests.
TermSuggestionBuilder termSuggest = termSuggestion("text")
.suggestMode(SuggestMode.ALWAYS)
.text("abcd");
Suggest suggest = searchSuggest("test", termSuggest);
assertSuggestionSize(suggest, 0, 0, "test");
assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
suggest = searchSuggest("test", termSuggest);
assertSuggestionSize(suggest, 0, 0, "test");
assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
index("test", "type1", "1", "text", "bar");
refresh();
suggest = searchSuggest("test", termSuggest);
assertSuggestionSize(suggest, 0, 0, "test");
assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
suggest = searchSuggest("test", termSuggest);
assertSuggestionSize(suggest, 0, 0, "test");
assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
}
public void testWithMultipleCommands() throws Exception {
assertAcked(prepareCreate("test").addMapping("typ1", "field1", "type=text", "field2", "type=text"));
ensureGreen();
@ -755,12 +783,7 @@ public class SuggestSearchIT extends ESIntegTestCase {
.put("index.analysis.filter.shingler.output_unigrams", true)).addMapping("type1", mappingBuilder));
ensureGreen();
index("test", "type1", "11", "foo", "bar");
index("test", "type1", "12", "foo", "bar");
index("test", "type1", "1", "name", "Just testing the suggestions api");
index("test", "type1", "2", "name", "An other title about equal length");
refresh();
// test phrase suggestion on completely empty index
SearchResponse searchResponse = client().prepareSearch()
.setSize(0)
.suggest(
@ -769,7 +792,44 @@ public class SuggestSearchIT extends ESIntegTestCase {
.get();
assertNoFailures(searchResponse);
assertSuggestion(searchResponse.getSuggest(), 0, 0, "did_you_mean", "testing suggestions");
Suggest suggest = searchResponse.getSuggest();
assertSuggestionSize(suggest, 0, 0, "did_you_mean");
assertThat(suggest.getSuggestion("did_you_mean").getEntries().get(0).getText().string(), equalTo("tetsting sugestion"));
index("test", "type1", "11", "foo", "bar");
index("test", "type1", "12", "foo", "bar");
index("test", "type1", "2", "name", "An other title about equal length");
refresh();
// test phrase suggestion but nothing matches
searchResponse = client().prepareSearch()
.setSize(0)
.suggest(
new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
phraseSuggestion("name").maxErrors(5.0f)))
.get();
assertNoFailures(searchResponse);
suggest = searchResponse.getSuggest();
assertSuggestionSize(suggest, 0, 0, "did_you_mean");
assertThat(suggest.getSuggestion("did_you_mean").getEntries().get(0).getText().string(), equalTo("tetsting sugestion"));
// finally indexing a document that will produce some meaningful suggestion
index("test", "type1", "1", "name", "Just testing the suggestions api");
refresh();
searchResponse = client().prepareSearch()
.setSize(0)
.suggest(
new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
phraseSuggestion("name").maxErrors(5.0f)))
.get();
assertNoFailures(searchResponse);
suggest = searchResponse.getSuggest();
assertSuggestionSize(suggest, 0, 3, "did_you_mean");
assertSuggestion(suggest, 0, 0, "did_you_mean", "testing suggestions");
}
/**