Fix suggestions for empty indices (#42927)

Currently suggesters return null values on empty shards. Usually this gets replaced
by results from other non-epmty shards, but if the index is completely epmty (e.g. after
creation) the search responses "suggest" is also "null" and we don't render a corresponding
output in the REST response. This is an irritating edge case that requires special handling on
the user side (see #42473) and should be fixed.

This change makes sure every suggester type (completion, terms, phrase) returns at least an
empty skeleton suggestion output, even for empty shards. This way, even if we don't find
any suggestions anywhere, we still return and output the empty suggestion.

Closes #42473

plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java

@@ -25,6 +25,7 @@ import org.elasticsearch.common.text.Text;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggester;
 
+import java.io.IOException;
 import java.util.Locale;
 
 public class CustomSuggester extends Suggester<CustomSuggestionContext> {
@@ -35,15 +36,12 @@ public class CustomSuggester extends Suggester<CustomSuggestionContext> {
         String name,
         CustomSuggestionContext suggestion,
         IndexSearcher searcher,
-        CharsRefBuilder spare) {
+        CharsRefBuilder spare) throws IOException {
-
-        // Get the suggestion context
-        String text = suggestion.getText().utf8ToString();
 
         // create two suggestions with 12 and 123 appended
-        CustomSuggestion response = new CustomSuggestion(name, suggestion.getSize(), "suggestion-dummy-value");
+        CustomSuggestion response = emptySuggestion(name, suggestion, spare);
-
+        CustomSuggestion.Entry entry = response.getEntries().get(0);
-        CustomSuggestion.Entry entry = new CustomSuggestion.Entry(new Text(text), 0, text.length(), "entry-dummy-value");
+        String text = entry.getText().string();
 
         String firstOption =
             String.format(Locale.ROOT, "%s-%s-%s-%s", text, suggestion.getField(), suggestion.options.get("suffix"), "12");
@@ -55,8 +53,16 @@ public class CustomSuggester extends Suggester<CustomSuggestionContext> {
         CustomSuggestion.Entry.Option option123 = new CustomSuggestion.Entry.Option(new Text(secondOption), 0.8f, "option-dummy-value-2");
         entry.addOption(option123);
 
-        response.addTerm(entry);
+        return response;
+    }
 
+    @Override
+    protected CustomSuggestion emptySuggestion(String name, CustomSuggestionContext suggestion,
+            CharsRefBuilder spare) throws IOException {
+        String text = suggestion.getText().utf8ToString();
+        CustomSuggestion response = new CustomSuggestion(name, suggestion.getSize(), "suggestion-dummy-value");
+        CustomSuggestion.Entry entry = new CustomSuggestion.Entry(new Text(text), 0, text.length(), "entry-dummy-value");
+        response.addTerm(entry);
         return response;
     }
 }

server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java

@@ -21,6 +21,7 @@ package org.elasticsearch.action.search;
 
 import com.carrotsearch.hppc.IntArrayList;
 import com.carrotsearch.hppc.ObjectObjectHashMap;
+
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.CollectionStatistics;
 import org.apache.lucene.search.FieldDoc;

server/src/main/java/org/elasticsearch/search/suggest/Suggester.java

@@ -29,12 +29,15 @@ public abstract class Suggester<T extends SuggestionSearchContext.SuggestionCont
     protected abstract Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
         innerExecute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException;
 
+    protected abstract Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
+        emptySuggestion(String name, T suggestion, CharsRefBuilder spare) throws IOException;
+
     public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
                 execute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
-        // #3469 We want to ignore empty shards
 
+        // we only want to output an empty suggestion on empty shards
         if (searcher.getIndexReader().numDocs() == 0) {
-            return null;
+            return emptySuggestion(name, suggestion, spare);
         }
         return innerExecute(name, suggestion, searcher, spare);
     }

server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java

@@ -49,12 +49,7 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
             final CompletionSuggestionContext suggestionContext, final IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
         if (suggestionContext.getFieldType() != null) {
             final CompletionFieldMapper.CompletionFieldType fieldType = suggestionContext.getFieldType();
-            CompletionSuggestion completionSuggestion =
+            CompletionSuggestion completionSuggestion = emptySuggestion(name, suggestionContext, spare);
-                new CompletionSuggestion(name, suggestionContext.getSize(), suggestionContext.isSkipDuplicates());
-            spare.copyUTF8Bytes(suggestionContext.getText());
-            CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(
-                new Text(spare.toString()), 0, spare.length());
-            completionSuggestion.addTerm(completionSuggestEntry);
             int shardSize = suggestionContext.getShardSize() != null ? suggestionContext.getShardSize() : suggestionContext.getSize();
             TopSuggestGroupDocsCollector collector = new TopSuggestGroupDocsCollector(shardSize, suggestionContext.isSkipDuplicates());
             suggest(searcher, suggestionContext.toQuery(), collector);
@@ -71,7 +66,7 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
                 if (numResult++ < suggestionContext.getSize()) {
                     CompletionSuggestion.Entry.Option option = new CompletionSuggestion.Entry.Option(suggestDoc.doc,
                         new Text(suggestDoc.key.toString()), suggestDoc.score, contexts);
-                    completionSuggestEntry.addOption(option);
+                    completionSuggestion.getEntries().get(0).addOption(option);
                 } else {
                     break;
                 }
@@ -96,4 +91,14 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
             }
         }
     }
+
+    @Override
+    protected CompletionSuggestion emptySuggestion(String name, CompletionSuggestionContext suggestion, CharsRefBuilder spare)
+            throws IOException {
+        CompletionSuggestion completionSuggestion = new CompletionSuggestion(name, suggestion.getSize(), suggestion.isSkipDuplicates());
+        spare.copyUTF8Bytes(suggestion.getText());
+        CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(new Text(spare.toString()), 0, spare.length());
+        completionSuggestion.addTerm(completionSuggestEntry);
+        return completionSuggestion;
+    }
 }

server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java

@@ -156,4 +156,13 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
         spare.copyUTF8Bytes(suggestion.getText());
         return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore);
     }
+
+    @Override
+    protected Suggestion<? extends Entry<? extends Option>> emptySuggestion(String name, PhraseSuggestionContext suggestion,
+            CharsRefBuilder spare) throws IOException {
+        PhraseSuggestion phraseSuggestion = new PhraseSuggestion(name, suggestion.getSize());
+        spare.copyUTF8Bytes(suggestion.getText());
+        phraseSuggestion.addTerm(new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length()));
+        return phraseSuggestion;
+    }
 }

server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java

@@ -94,4 +94,16 @@ public final class TermSuggester extends Suggester<TermSuggestionContext> {
         }
 
     }
+
+    @Override
+    protected TermSuggestion emptySuggestion(String name, TermSuggestionContext suggestion, CharsRefBuilder spare) throws IOException {
+        TermSuggestion termSuggestion = new TermSuggestion(name, suggestion.getSize(), suggestion.getDirectSpellCheckerSettings().sort());
+        List<Token> tokens = queryTerms(suggestion, spare);
+        for (Token token : tokens) {
+            Text key = new Text(new BytesArray(token.term.bytes()));
+            TermSuggestion.Entry resultEntry = new TermSuggestion.Entry(key, token.startOffset, token.endOffset - token.startOffset);
+            termSuggestion.addTerm(resultEntry);
+        }
+        return termSuggestion;
+    }
 }

server/src/test/java/org/elasticsearch/search/SearchModuleTests.java

@@ -66,9 +66,12 @@ import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
 import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.term.TermSuggestion;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 import org.elasticsearch.test.ESTestCase;
@@ -173,6 +176,7 @@ public class SearchModuleTests extends ESTestCase {
         expectThrows(IllegalArgumentException.class, registryForPlugin(registersDupePipelineAggregation));
 
         SearchPlugin registersDupeRescorer = new SearchPlugin() {
+            @Override
             public List<RescorerSpec<?>> getRescorers() {
                 return singletonList(
                         new RescorerSpec<>(QueryRescorerBuilder.NAME, QueryRescorerBuilder::new, QueryRescorerBuilder::fromXContent));
@@ -525,6 +529,7 @@ public class SearchModuleTests extends ESTestCase {
     }
 
     private static class TestSuggester extends Suggester<SuggestionSearchContext.SuggestionContext> {
+
         @Override
         protected Suggestion<? extends Suggestion.Entry<? extends Suggestion.Entry.Option>> innerExecute(
                 String name,
@@ -533,6 +538,12 @@ public class SearchModuleTests extends ESTestCase {
                 CharsRefBuilder spare) throws IOException {
             return null;
         }
+
+        @Override
+        protected Suggestion<? extends Entry<? extends Option>> emptySuggestion(String name, SuggestionContext suggestion,
+                CharsRefBuilder spare) throws IOException {
+            return null;
+        }
     }
 
     private static class TestSuggestionBuilder extends SuggestionBuilder<TestSuggestionBuilder> {

server/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java

@@ -357,6 +357,26 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         }
     }
 
+    /**
+     * Suggestions run on an empty index should return a suggest element as part of the response. See #42473 for details.
+     */
+    public void testSuggestEmptyIndex() throws IOException, InterruptedException {
+        final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
+        createIndexAndMapping(mapping);
+
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("v");
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
+                .setFetchSource("a", "b").get();
+        Suggest suggest = searchResponse.getSuggest();
+        assertNotNull(suggest);
+        CompletionSuggestion completionSuggestion = suggest.getSuggestion("foo");
+        CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0);
+        assertEquals("v", options.getText().string());
+        assertEquals(1, options.getLength());
+        assertEquals(0, options.getOffset());
+        assertEquals(0, options.options.size());
+    }
+
     public void testThatWeightsAreWorking() throws Exception {
         createIndexAndMapping(completionMappingBuilder);
 

server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java

@@ -345,6 +345,34 @@ public class SuggestSearchIT extends ESIntegTestCase {
         assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
     }
 
+    public void testEmptyIndex() throws Exception {
+        assertAcked(prepareCreate("test").addMapping("type1", "text", "type=text"));
+        ensureGreen();
+
+        // use SuggestMode.ALWAYS, otherwise the results can vary between requests.
+        TermSuggestionBuilder termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS)
+                .text("abcd");
+        Suggest suggest = searchSuggest("test", termSuggest);
+        assertSuggestionSize(suggest, 0, 0, "test");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+
+        suggest = searchSuggest("test", termSuggest);
+        assertSuggestionSize(suggest, 0, 0, "test");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+
+        index("test", "type1", "1", "text", "bar");
+        refresh();
+
+        suggest = searchSuggest("test", termSuggest);
+        assertSuggestionSize(suggest, 0, 0, "test");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+
+        suggest = searchSuggest("test", termSuggest);
+        assertSuggestionSize(suggest, 0, 0, "test");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+    }
+
     public void testWithMultipleCommands() throws Exception {
         assertAcked(prepareCreate("test").addMapping("typ1", "field1", "type=text", "field2", "type=text"));
         ensureGreen();
@@ -755,12 +783,7 @@ public class SuggestSearchIT extends ESIntegTestCase {
                 .put("index.analysis.filter.shingler.output_unigrams", true)).addMapping("type1", mappingBuilder));
         ensureGreen();
 
-        index("test", "type1", "11", "foo", "bar");
+        // test phrase suggestion on completely empty index
-        index("test", "type1", "12", "foo", "bar");
-        index("test", "type1", "1", "name", "Just testing the suggestions api");
-        index("test", "type1", "2", "name", "An other title about equal length");
-        refresh();
-
         SearchResponse searchResponse = client().prepareSearch()
                 .setSize(0)
                 .suggest(
@@ -769,7 +792,44 @@ public class SuggestSearchIT extends ESIntegTestCase {
                 .get();
 
         assertNoFailures(searchResponse);
-        assertSuggestion(searchResponse.getSuggest(), 0, 0, "did_you_mean", "testing suggestions");
+        Suggest suggest = searchResponse.getSuggest();
+        assertSuggestionSize(suggest, 0, 0, "did_you_mean");
+        assertThat(suggest.getSuggestion("did_you_mean").getEntries().get(0).getText().string(), equalTo("tetsting sugestion"));
+
+
+        index("test", "type1", "11", "foo", "bar");
+        index("test", "type1", "12", "foo", "bar");
+        index("test", "type1", "2", "name", "An other title about equal length");
+        refresh();
+
+        // test phrase suggestion but nothing matches
+        searchResponse = client().prepareSearch()
+                .setSize(0)
+                .suggest(
+                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
+                                phraseSuggestion("name").maxErrors(5.0f)))
+                .get();
+
+        assertNoFailures(searchResponse);
+        suggest = searchResponse.getSuggest();
+        assertSuggestionSize(suggest, 0, 0, "did_you_mean");
+        assertThat(suggest.getSuggestion("did_you_mean").getEntries().get(0).getText().string(), equalTo("tetsting sugestion"));
+
+        // finally indexing a document that will produce some meaningful suggestion
+        index("test", "type1", "1", "name", "Just testing the suggestions api");
+        refresh();
+
+        searchResponse = client().prepareSearch()
+                .setSize(0)
+                .suggest(
+                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
+                                phraseSuggestion("name").maxErrors(5.0f)))
+                .get();
+
+        assertNoFailures(searchResponse);
+        suggest = searchResponse.getSuggest();
+        assertSuggestionSize(suggest, 0, 3, "did_you_mean");
+        assertSuggestion(suggest, 0, 0, "did_you_mean", "testing suggestions");
     }
 
     /**