Adding method to build SuggestionContext to PhraseSuggestionBuilder

This adds a build method for the SuggestionContext to the PhraseSuggestionBuilder
and another one that creates the SuggestionSearchContext to the top level
SuggestBuilder. Also adding tests that make sure the current way of parsing
xContent to a SuggestionContext is reflected in the output the builders create.

core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java

@@ -142,8 +142,7 @@ public class TransportSuggestAction extends TransportBroadcastAction<SuggestRequ
                 if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
                     throw new IllegalArgumentException("suggest content missing");
                 }
-                final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.mapperService(),
+                final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.newQueryShardContext());
-                        indexService.fieldData(), request.shardId().getIndexName(), request.shardId().id());
                 final Suggest result = suggestPhase.execute(context, searcher.searcher());
                 return new ShardSuggestResponse(request.shardId(), result);
             }

core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java

@@ -39,7 +39,7 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
+import org.elasticsearch.search.suggest.phrase.SmoothingModel;
 import org.elasticsearch.tasks.Task;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;

core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java

@@ -38,7 +38,7 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
+import org.elasticsearch.search.suggest.phrase.SmoothingModel;
 import org.elasticsearch.tasks.Task;
 import org.joda.time.ReadableInstant;
 

core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java

@@ -19,12 +19,12 @@
 
 package org.elasticsearch.indices.query;
 
-import java.util.Map;
-
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryParser;
 
+import java.util.Map;
+
 public class IndicesQueriesRegistry extends AbstractComponent {
     private Map<String, QueryParser<?>> queryParsers;
 

core/src/main/java/org/elasticsearch/search/SearchService.java

@@ -751,7 +751,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         if (source.rescores() != null) {
             try {
                 for (RescoreBuilder<?> rescore : source.rescores()) {
-                    context.addRescore(rescore.build(context.getQueryShardContext()));
+                    context.addRescore(rescore.build(queryShardContext));
                 }
             } catch (IOException e) {
                 throw new SearchContextException(context, "failed to create RescoreSearchContext", e);
@@ -776,7 +776,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         if (source.highlighter() != null) {
             HighlightBuilder highlightBuilder = source.highlighter();
             try {
-                context.highlight(highlightBuilder.build(context.getQueryShardContext()));
+                context.highlight(highlightBuilder.build(queryShardContext));
             } catch (IOException e) {
                 throw new SearchContextException(context, "failed to create SearchContextHighlighter", e);
             }

core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java

@@ -26,9 +26,12 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -137,6 +140,21 @@ public class SuggestBuilder extends ToXContentToBytes implements Writeable<Sugge
         return suggestBuilder;
     }
 
+    public SuggestionSearchContext build(QueryShardContext context) throws IOException {
+        SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext();
+        for (SuggestionBuilder<?> suggestionBuilder : suggestions) {
+            SuggestionContext suggestionContext = suggestionBuilder.build(context);
+            if (suggestionContext.getText() == null) {
+                if (globalText == null) {
+                    throw new IllegalArgumentException("The required text option is missing");
+                }
+                suggestionContext.setText(BytesRefs.toBytesRef(globalText));
+            }
+            suggestionSearchContext.addSuggestion(suggestionBuilder.name(), suggestionContext);
+        }
+        return suggestionSearchContext;
+    }
+
     @Override
     public SuggestBuilder readFrom(StreamInput in) throws IOException {
         final SuggestBuilder builder = new SuggestBuilder();

core/src/main/java/org/elasticsearch/search/suggest/SuggestContextParser.java

@@ -19,12 +19,11 @@
 package org.elasticsearch.search.suggest;
 
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.mapper.MapperService;
 
 import java.io.IOException;
 
 public interface SuggestContextParser {
-    SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService indexFieldDataService) throws IOException;
+    SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException;
 
 }

core/src/main/java/org/elasticsearch/search/suggest/SuggestParseElement.java

@@ -21,8 +21,8 @@ package org.elasticsearch.search.suggest;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
@@ -44,14 +44,13 @@ public final class SuggestParseElement implements SearchParseElement {
 
     @Override
     public void parse(XContentParser parser, SearchContext context) throws Exception {
-        SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.mapperService(), context.fieldData(),
+        SuggestionSearchContext suggestionSearchContext = parseInternal(parser, context.getQueryShardContext());
-                context.shardTarget().index(), context.shardTarget().shardId());
         context.suggest(suggestionSearchContext);
     }
 
-    public SuggestionSearchContext parseInternal(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService,
+    public SuggestionSearchContext parseInternal(XContentParser parser, QueryShardContext shardContext) throws IOException {
-                                                 String index, int shardId) throws IOException {
         SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext();
+        MapperService mapperService = shardContext.getMapperService();
 
         BytesRef globalText = null;
         String fieldName = null;
@@ -95,10 +94,20 @@ public final class SuggestParseElement implements SearchParseElement {
                             throw new IllegalArgumentException("Suggester[" + fieldName + "] not supported");
                         }
                         final SuggestContextParser contextParser = suggesters.get(fieldName).getContextParser();
-                        suggestionContext = contextParser.parse(parser, mapperService, fieldDataService);
+                        suggestionContext = contextParser.parse(parser, shardContext);
                     }
                 }
                 if (suggestionContext != null) {
+                    if (suggestText != null) {
+                        suggestionContext.setText(suggestText);
+                    }
+                    if (prefix != null) {
+                        suggestionContext.setPrefix(prefix);
+                    }
+                    if (regex != null) {
+                        suggestionContext.setRegex(regex);
+                    }
+
                     if (suggestText != null && prefix == null) {
                         suggestionContext.setPrefix(suggestText);
                         suggestionContext.setText(suggestText);
@@ -110,6 +119,8 @@ public final class SuggestParseElement implements SearchParseElement {
                         suggestionContext.setText(regex);
                     }
                     suggestionContexts.put(suggestionName, suggestionContext);
+                } else {
+                    throw new IllegalArgumentException("suggestion context could not be parsed correctly");
                 }
             }
         }
@@ -117,9 +128,6 @@ public final class SuggestParseElement implements SearchParseElement {
         for (Map.Entry<String, SuggestionContext> entry : suggestionContexts.entrySet()) {
             String suggestionName = entry.getKey();
             SuggestionContext suggestionContext = entry.getValue();
-
-            suggestionContext.setShard(shardId);
-            suggestionContext.setIndex(index);
             SuggestUtils.verifySuggestion(mapperService, globalText, suggestionContext);
             suggestionSearchContext.addSuggestion(suggestionName, suggestionContext);
         }

core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java

@@ -271,10 +271,10 @@ public final class SuggestUtils {
            return false;
         }
         return true;
-
     }
 
 
+
     public static void verifySuggestion(MapperService mapperService, BytesRef globalText, SuggestionContext suggestion) {
         // Verify options and set defaults
         if (suggestion.getField() == null) {
@@ -294,7 +294,6 @@ public final class SuggestUtils {
         }
     }
 
-
     public static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) {
         if (analyzer instanceof NamedAnalyzer) {
             analyzer = ((NamedAnalyzer)analyzer).analyzer();

core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java

@@ -20,8 +20,6 @@ package org.elasticsearch.search.suggest;
 
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.ExtensionPoint;
-import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.suggest.completion.CompletionSuggester;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggester;
 import org.elasticsearch.search.suggest.term.TermSuggester;
@@ -42,21 +40,17 @@ public final class Suggesters extends ExtensionPoint.ClassMap<Suggester> {
         this(Collections.emptyMap());
     }
 
+    @Inject
     public Suggesters(Map<String, Suggester> suggesters) {
         super("suggester", Suggester.class, new HashSet<>(Arrays.asList("phrase", "term", "completion")), Suggesters.class, SuggestParseElement.class, SuggestPhase.class);
-        this.parsers = Collections.unmodifiableMap(suggesters);
+        this.parsers = Collections.unmodifiableMap(addBuildIns(suggesters));
     }
 
-    @Inject
+    private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters) {
-    public Suggesters(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) {
-        this(addBuildIns(suggesters, scriptService, indexServices));
-    }
-
-    private static Map<String, Suggester> addBuildIns(Map<String, Suggester> suggesters, ScriptService scriptService, IndicesService indexServices) {
         final Map<String, Suggester> map = new HashMap<>();
-        map.put("phrase", new PhraseSuggester(scriptService, indexServices));
+        map.put("phrase", PhraseSuggester.PROTOTYPE);
-        map.put("term", new TermSuggester());
+        map.put("term", TermSuggester.PROTOTYPE);
-        map.put("completion", new CompletionSuggester());
+        map.put("completion", CompletionSuggester.PROTOTYPE);
         map.putAll(suggesters);
         return map;
     }

core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java

@@ -19,15 +19,20 @@
 
 package org.elasticsearch.search.suggest;
 
+import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -192,6 +197,56 @@ public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends
 
     protected abstract SuggestionBuilder<T> innerFromXContent(QueryParseContext parseContext, String name) throws IOException;
 
+    protected abstract SuggestionContext build(QueryShardContext context) throws IOException;
+
+    /**
+     * Transfers the text, prefix, regex, analyzer, fieldname, size and shard size settings from the
+     * original {@link SuggestionBuilder} to the target {@link SuggestionContext}
+     */
+    protected void populateCommonFields(MapperService mapperService,
+            SuggestionSearchContext.SuggestionContext suggestionContext) throws IOException {
+
+        if (analyzer != null) {
+            Analyzer luceneAnalyzer = mapperService.analysisService().analyzer(analyzer);
+            if (luceneAnalyzer == null) {
+                throw new IllegalArgumentException("Analyzer [" + luceneAnalyzer + "] doesn't exists");
+            }
+            suggestionContext.setAnalyzer(luceneAnalyzer);
+        }
+
+        if (fieldname != null) {
+            suggestionContext.setField(fieldname);
+        }
+
+        if (size != null) {
+            suggestionContext.setSize(size);
+        }
+
+        if (shardSize != null) {
+            suggestionContext.setShardSize(shardSize);
+        } else {
+            // if no shard size is set in builder, use size (or at least 5)
+            suggestionContext.setShardSize(Math.max(suggestionContext.getSize(), 5));
+        }
+
+        if (text != null) {
+            suggestionContext.setText(BytesRefs.toBytesRef(text));
+        }
+        if (prefix != null) {
+            suggestionContext.setPrefix(BytesRefs.toBytesRef(prefix));
+        }
+        if (regex != null) {
+            suggestionContext.setRegex(BytesRefs.toBytesRef(regex));
+        }
+        if (text != null && prefix == null) {
+            suggestionContext.setPrefix(BytesRefs.toBytesRef(text));
+        } else if (text == null && prefix != null) {
+            suggestionContext.setText(BytesRefs.toBytesRef(prefix));
+        } else if (text == null && regex != null) {
+            suggestionContext.setText(BytesRefs.toBytesRef(regex));
+        }
+    }
+
     private String getSuggesterName() {
         //default impl returns the same as writeable name, but we keep the distinction between the two just to make sure
         return getWriteableName();

core/src/main/java/org/elasticsearch/search/suggest/SuggestionSearchContext.java

@@ -20,6 +20,7 @@ package org.elasticsearch.search.suggest;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -37,18 +38,22 @@ public class SuggestionSearchContext {
         return suggestions;
     }
 
-    public static class SuggestionContext {
+    public abstract static class SuggestionContext {
 
         private BytesRef text;
         private BytesRef prefix;
         private BytesRef regex;
-        private final Suggester suggester;
         private String field;
         private Analyzer analyzer;
         private int size = 5;
         private int shardSize = -1;
-        private int shardId;
+        private QueryShardContext shardContext;
-        private String index;
+        private Suggester<?> suggester;
+
+        protected SuggestionContext(Suggester<?> suggester, QueryShardContext shardContext) {
+            this.suggester = suggester;
+            this.shardContext = shardContext;
+        }
 
         public BytesRef getText() {
             return text;
@@ -74,12 +79,8 @@ public class SuggestionSearchContext {
             this.regex = regex;
         }
 
-        public SuggestionContext(Suggester suggester) {
-            this.suggester = suggester;
-        }
-
         public Suggester<SuggestionContext> getSuggester() {
-            return this.suggester;
+            return ((Suggester<SuggestionContext>) suggester);
         }
 
         public Analyzer getAnalyzer() {
@@ -120,20 +121,8 @@ public class SuggestionSearchContext {
             this.shardSize = shardSize;
         }
 
-        public void setShard(int shardId) {
+        public QueryShardContext getShardContext() {
-            this.shardId = shardId;
+            return this.shardContext;
-        }
-
-        public void setIndex(String index) {
-            this.index = index;
-        }
-        
-        public String getIndex() {
-            return index;
-        }
-        
-        public int getShard() {
-            return shardId;
         }
     }
 

core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestParser.java

@@ -20,17 +20,16 @@ package org.elasticsearch.search.suggest.completion;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.RegexpFlag;
 import org.elasticsearch.search.suggest.SuggestContextParser;
 import org.elasticsearch.search.suggest.SuggestUtils.Fields;
@@ -135,8 +134,9 @@ public class CompletionSuggestParser implements SuggestContextParser {
     }
 
     @Override
-    public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
+    public SuggestionSearchContext.SuggestionContext parse(XContentParser parser,  QueryShardContext shardContext) throws IOException {
-        final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(completionSuggester, mapperService, fieldDataService);
+        MapperService mapperService = shardContext.getMapperService();
+        final CompletionSuggestionContext suggestion = new CompletionSuggestionContext(shardContext);
         final ContextAndSuggest contextAndSuggest = new ContextAndSuggest(mapperService);
         TLP_PARSER.parse(parser, suggestion, contextAndSuggest);
         final XContentParser contextParser = contextAndSuggest.contextParser;

core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java

@@ -34,7 +34,9 @@ import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.fielddata.AtomicFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.SuggestContextParser;
 import org.elasticsearch.search.suggest.Suggester;
@@ -51,6 +53,8 @@ import java.util.Set;
 
 public class CompletionSuggester extends Suggester<CompletionSuggestionContext> {
 
+    public static final CompletionSuggester PROTOTYPE = new CompletionSuggester();
+
     @Override
     public SuggestContextParser getContextParser() {
         return new CompletionSuggestParser(this);
@@ -86,9 +90,11 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
                 final LeafReaderContext subReaderContext = leaves.get(readerIndex);
                 final int subDocId = suggestDoc.doc - subReaderContext.docBase;
                 for (String field : payloadFields) {
-                    MappedFieldType payloadFieldType = suggestionContext.getMapperService().fullName(field);
+                    MapperService mapperService = suggestionContext.getShardContext().getMapperService();
+                    MappedFieldType payloadFieldType = mapperService.fullName(field);
                     if (payloadFieldType != null) {
-                        final AtomicFieldData data = suggestionContext.getIndexFieldDataService().getForField(payloadFieldType)
+                        QueryShardContext shardContext = suggestionContext.getShardContext();
+                        final AtomicFieldData data = shardContext.getForField(payloadFieldType)
                                 .load(subReaderContext);
                         final ScriptDocValues scriptValues = data.getScriptValues();
                         scriptValues.setNextDocId(subDocId);

core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java

@@ -28,8 +28,10 @@ import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.RegexpFlag;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
 import org.elasticsearch.search.suggest.completion.context.GeoQueryContext;
 
@@ -372,9 +374,16 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
 
     @Override
     protected CompletionSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String name) throws IOException {
+        // NORELEASE
         return new CompletionSuggestionBuilder(name);
     }
 
+    @Override
+    protected SuggestionContext build(QueryShardContext context) throws IOException {
+        // NORELEASE
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public String getWriteableName() {
         return SUGGESTION_NAME;

core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java

@@ -20,10 +20,8 @@ package org.elasticsearch.search.suggest.completion;
 
 import org.apache.lucene.search.suggest.document.CompletionQuery;
 import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.core.CompletionFieldMapper;
-import org.elasticsearch.search.suggest.Suggester;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
 import org.elasticsearch.search.suggest.completion.context.ContextMappings;
@@ -39,20 +37,16 @@ import java.util.Set;
  */
 public class CompletionSuggestionContext extends SuggestionSearchContext.SuggestionContext {
 
+    protected CompletionSuggestionContext(QueryShardContext shardContext) {
+        super(CompletionSuggester.PROTOTYPE, shardContext);
+    }
+
     private CompletionFieldMapper.CompletionFieldType fieldType;
     private CompletionSuggestionBuilder.FuzzyOptionsBuilder fuzzyOptionsBuilder;
     private CompletionSuggestionBuilder.RegexOptionsBuilder regexOptionsBuilder;
     private Map<String, List<ContextMapping.QueryContext>> queryContexts = Collections.emptyMap();
-    private final MapperService mapperService;
-    private final IndexFieldDataService indexFieldDataService;
     private Set<String> payloadFields = Collections.emptySet();
 
-    CompletionSuggestionContext(Suggester suggester, MapperService mapperService, IndexFieldDataService indexFieldDataService) {
-        super(suggester);
-        this.indexFieldDataService = indexFieldDataService;
-        this.mapperService = mapperService;
-    }
-
     CompletionFieldMapper.CompletionFieldType getFieldType() {
         return this.fieldType;
     }
@@ -73,15 +67,6 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest
         this.queryContexts = queryContexts;
     }
 
-
-    MapperService getMapperService() {
-        return mapperService;
-    }
-
-    IndexFieldDataService getIndexFieldDataService() {
-        return indexFieldDataService;
-    }
-
     void setPayloadFields(Set<String> fields) {
         this.payloadFields = fields;
     }

core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java

@@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.SuggestUtils;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.CandidateGenerator;
 
@@ -349,8 +348,7 @@ public final class DirectCandidateGeneratorBuilder
         return replaceField(tmpFieldName.iterator().next(), tempGenerator);
     }
 
-    public PhraseSuggestionContext.DirectCandidateGenerator build(QueryShardContext context) throws IOException {
+    public PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException {
-        MapperService mapperService = context.getMapperService();
         PhraseSuggestionContext.DirectCandidateGenerator generator = new PhraseSuggestionContext.DirectCandidateGenerator();
         generator.setField(this.field);
         transferIfNotNull(this.size, generator::size);

core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java

@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.phrase;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive
+ * smoothing</a> model.
+ * <p>
+ * See <a
+ * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
+ * Smoothing</a> for details.
+ * </p>
+ */
+public final class Laplace extends SmoothingModel {
+    private double alpha = DEFAULT_LAPLACE_ALPHA;
+    private static final String NAME = "laplace";
+    private static final ParseField ALPHA_FIELD = new ParseField("alpha");
+    static final ParseField PARSE_FIELD = new ParseField(NAME);
+    /**
+     * Default alpha parameter for laplace smoothing
+     */
+    public static final double DEFAULT_LAPLACE_ALPHA = 0.5;
+    public static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA);
+
+    /**
+     * Creates a Laplace smoothing model.
+     *
+     */
+    public Laplace(double alpha) {
+        this.alpha = alpha;
+    }
+
+    /**
+     * @return the laplace model alpha parameter
+     */
+    public double getAlpha() {
+        return this.alpha;
+    }
+
+    @Override
+    protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(ALPHA_FIELD.getPreferredName(), alpha);
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeDouble(alpha);
+    }
+
+    @Override
+    public SmoothingModel readFrom(StreamInput in) throws IOException {
+        return new Laplace(in.readDouble());
+    }
+
+    @Override
+    protected boolean doEquals(SmoothingModel other) {
+        Laplace otherModel = (Laplace) other;
+        return Objects.equals(alpha, otherModel.alpha);
+    }
+
+    @Override
+    protected final int doHashCode() {
+        return Objects.hash(alpha);
+    }
+
+    @Override
+    public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        XContentParser.Token token;
+        String fieldName = null;
+        double alpha = DEFAULT_LAPLACE_ALPHA;
+        while ((token = parser.nextToken()) != Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            }
+            if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) {
+                alpha = parser.doubleValue();
+            }
+        }
+        return new Laplace(alpha);
+    }
+
+    @Override
+    public WordScorerFactory buildWordScorerFactory() {
+        return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
+                -> new LaplaceScorer(reader, terms,  field, realWordLikelyhood, separator, alpha);
+    }
+}

core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java

@@ -0,0 +1,176 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.phrase;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Linear interpolation smoothing model.
+ * <p>
+ * See <a
+ * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
+ * Smoothing</a> for details.
+ * </p>
+ */
+public final class LinearInterpolation extends SmoothingModel {
+    private static final String NAME = "linear";
+    public static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1);
+    private final double trigramLambda;
+    private final double bigramLambda;
+    private final double unigramLambda;
+    static final ParseField PARSE_FIELD = new ParseField(NAME);
+    private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda");
+    private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda");
+    private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda");
+
+    /**
+     * Creates a linear interpolation smoothing model.
+     *
+     * Note: the lambdas must sum up to one.
+     *
+     * @param trigramLambda
+     *            the trigram lambda
+     * @param bigramLambda
+     *            the bigram lambda
+     * @param unigramLambda
+     *            the unigram lambda
+     */
+    public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) {
+        double sum = trigramLambda + bigramLambda + unigramLambda;
+        if (Math.abs(sum - 1.0) > 0.001) {
+            throw new IllegalArgumentException("linear smoothing lambdas must sum to 1");
+        }
+        this.trigramLambda = trigramLambda;
+        this.bigramLambda = bigramLambda;
+        this.unigramLambda = unigramLambda;
+    }
+
+    public double getTrigramLambda() {
+        return this.trigramLambda;
+    }
+
+    public double getBigramLambda() {
+        return this.bigramLambda;
+    }
+
+    public double getUnigramLambda() {
+        return this.unigramLambda;
+    }
+
+    @Override
+    protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda);
+        builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda);
+        builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda);
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeDouble(trigramLambda);
+        out.writeDouble(bigramLambda);
+        out.writeDouble(unigramLambda);
+    }
+
+    @Override
+    public LinearInterpolation readFrom(StreamInput in) throws IOException {
+        return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble());
+    }
+
+    @Override
+    protected boolean doEquals(SmoothingModel other) {
+        final LinearInterpolation otherModel = (LinearInterpolation) other;
+        return Objects.equals(trigramLambda, otherModel.trigramLambda) &&
+                Objects.equals(bigramLambda, otherModel.bigramLambda) &&
+                Objects.equals(unigramLambda, otherModel.unigramLambda);
+    }
+
+    @Override
+    protected final int doHashCode() {
+        return Objects.hash(trigramLambda, bigramLambda, unigramLambda);
+    }
+
+    @Override
+    public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        XContentParser.Token token;
+        String fieldName = null;
+        double trigramLambda = 0.0;
+        double bigramLambda = 0.0;
+        double unigramLambda = 0.0;
+        ParseFieldMatcher matcher = parseContext.parseFieldMatcher();
+        while ((token = parser.nextToken()) != Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (matcher.match(fieldName, TRIGRAM_FIELD)) {
+                    trigramLambda = parser.doubleValue();
+                    if (trigramLambda < 0) {
+                        throw new IllegalArgumentException("trigram_lambda must be positive");
+                    }
+                } else if (matcher.match(fieldName, BIGRAM_FIELD)) {
+                    bigramLambda = parser.doubleValue();
+                    if (bigramLambda < 0) {
+                        throw new IllegalArgumentException("bigram_lambda must be positive");
+                    }
+                } else if (matcher.match(fieldName, UNIGRAM_FIELD)) {
+                    unigramLambda = parser.doubleValue();
+                    if (unigramLambda < 0) {
+                        throw new IllegalArgumentException("unigram_lambda must be positive");
+                    }
+                } else {
+                    throw new IllegalArgumentException(
+                            "suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
+            }
+        }
+        return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda);
+    }
+
+    @Override
+    public WordScorerFactory buildWordScorerFactory() {
+        return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) ->
+                    new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda,
+                        unigramLambda);
+    }
+}

core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestParser.java

@@ -26,17 +26,16 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.search.suggest.SuggestContextParser;
 import org.elasticsearch.search.suggest.SuggestUtils;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
 
 import java.io.IOException;
@@ -51,8 +50,10 @@ public final class PhraseSuggestParser implements SuggestContextParser {
     }
 
     @Override
-    public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
+    public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException {
-        PhraseSuggestionContext suggestion = new PhraseSuggestionContext(suggester);
+        MapperService mapperService = shardContext.getMapperService();
+        ScriptService scriptService = shardContext.getScriptService();
+        PhraseSuggestionContext suggestion = new PhraseSuggestionContext(shardContext);
         ParseFieldMatcher parseFieldMatcher = mapperService.getIndexSettings().getParseFieldMatcher();
         XContentParser.Token token;
         String fieldName = null;
@@ -135,7 +136,7 @@ public final class PhraseSuggestParser implements SuggestContextParser {
                                 throw new IllegalArgumentException("suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]");
                             }
                             Template template = Template.parse(parser, parseFieldMatcher);
-                            CompiledScript compiledScript = suggester.scriptService().compile(template, ScriptContext.Standard.SEARCH, Collections.emptyMap());
+                            CompiledScript compiledScript = scriptService.compile(template, ScriptContext.Standard.SEARCH, Collections.emptyMap());
                             suggestion.setCollateQueryScript(compiledScript);
                         } else if ("params".equals(fieldName)) {
                             suggestion.setCollateScriptParams(parser.map());
@@ -199,9 +200,6 @@ public final class PhraseSuggestParser implements SuggestContextParser {
                 suggestion.addGenerator(generator);
             }
         }
-
-
-
         return suggestion;
     }
 

core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java

@@ -31,9 +31,7 @@ import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.query.ParsedQuery;
-import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptService;
@@ -44,6 +42,7 @@ import org.elasticsearch.search.suggest.SuggestContextParser;
 import org.elasticsearch.search.suggest.SuggestUtils;
 import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result;
 
 import java.io.IOException;
@@ -54,13 +53,8 @@ import java.util.Map;
 public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
     private final BytesRef SEPARATOR = new BytesRef(" ");
     private static final String SUGGESTION_TEMPLATE_VAR_NAME = "suggestion";
-    private final ScriptService scriptService;
-    private final IndicesService indicesService;
 
-    public PhraseSuggester(ScriptService scriptService, IndicesService indicesService) {
+    public static final PhraseSuggester PROTOTYPE = new PhraseSuggester();
-        this.scriptService = scriptService;
-        this.indicesService = indicesService;
-    }
 
     /*
      * More Ideas:
@@ -118,10 +112,10 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
                     // from the index for a correction, collateMatch is updated
                     final Map<String, Object> vars = suggestion.getCollateScriptParams();
                     vars.put(SUGGESTION_TEMPLATE_VAR_NAME, spare.toString());
+                    ScriptService scriptService = suggestion.getShardContext().getScriptService();
                     final ExecutableScript executable = scriptService.executable(collateScript, vars);
                     final BytesReference querySource = (BytesReference) executable.run();
-                    IndexService indexService = indicesService.indexService(suggestion.getIndex());
+                    final ParsedQuery parsedQuery = suggestion.getShardContext().parse(querySource);
-                    final ParsedQuery parsedQuery = indexService.newQueryShardContext().parse(querySource);
                     collateMatch = Lucene.exists(searcher, parsedQuery.query());
                 }
                 if (!collateMatch && !collatePrune) {
@@ -145,15 +139,11 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
         return response;
     }
 
-    private PhraseSuggestion.Entry buildResultEntry(PhraseSuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) {
+    private PhraseSuggestion.Entry buildResultEntry(SuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) {
         spare.copyUTF8Bytes(suggestion.getText());
         return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore);
     }
 
-    ScriptService scriptService() {
-        return scriptService;
-    }
-
     @Override
     public SuggestContextParser getContextParser() {
         return new PhraseSuggestParser(this);

core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java

@@ -19,27 +19,32 @@
 package org.elasticsearch.search.suggest.phrase;
 
 
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.Template;
+import org.elasticsearch.search.suggest.SuggestUtils;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
-import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -239,7 +244,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
 
     /**
      * Sets an explicit smoothing model used for this suggester. The default is
-     * {@link PhraseSuggestionBuilder.StupidBackoff}.
+     * {@link StupidBackoff}.
      */
     public PhraseSuggestionBuilder smoothingModel(SmoothingModel model) {
         this.model = model;
@@ -254,6 +259,9 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
     }
 
     public PhraseSuggestionBuilder tokenLimit(int tokenLimit) {
+        if (tokenLimit <= 0) {
+            throw new IllegalArgumentException("token_limit must be >= 1");
+        }
         this.tokenLimit = tokenLimit;
         return this;
     }
@@ -389,413 +397,6 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
         return builder;
     }
 
-    /**
-     * Creates a new {@link DirectCandidateGeneratorBuilder}
-     *
-     * @param field
-     *            the field this candidate generator operates on.
-     */
-    public static DirectCandidateGeneratorBuilder candidateGenerator(String field) {
-        return new DirectCandidateGeneratorBuilder(field);
-    }
-
-    /**
-     * A "stupid-backoff" smoothing model simialr to <a
-     * href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's
-     * Backoff</a>. This model is used as the default if no model is configured.
-     * <p>
-     * See <a
-     * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
-     * Smoothing</a> for details.
-     * </p>
-     */
-    public static final class StupidBackoff extends SmoothingModel {
-        /**
-         * Default discount parameter for {@link StupidBackoff} smoothing
-         */
-        public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4;
-        public static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT);
-        private double discount = DEFAULT_BACKOFF_DISCOUNT;
-        private static final String NAME = "stupid_backoff";
-        private static final ParseField DISCOUNT_FIELD = new ParseField("discount");
-        private static final ParseField PARSE_FIELD = new ParseField(NAME);
-
-        /**
-         * Creates a Stupid-Backoff smoothing model.
-         *
-         * @param discount
-         *            the discount given to lower order ngrams if the higher order ngram doesn't exits
-         */
-        public StupidBackoff(double discount) {
-            this.discount = discount;
-        }
-
-        /**
-         * @return the discount parameter of the model
-         */
-        public double getDiscount() {
-            return this.discount;
-        }
-
-        @Override
-        protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(DISCOUNT_FIELD.getPreferredName(), discount);
-            return builder;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeDouble(discount);
-        }
-
-        @Override
-        public StupidBackoff readFrom(StreamInput in) throws IOException {
-            return new StupidBackoff(in.readDouble());
-        }
-
-        @Override
-        protected boolean doEquals(SmoothingModel other) {
-            StupidBackoff otherModel = (StupidBackoff) other;
-            return Objects.equals(discount, otherModel.discount);
-        }
-
-        @Override
-        protected final int doHashCode() {
-            return Objects.hash(discount);
-        }
-
-        @Override
-        public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
-            XContentParser parser = parseContext.parser();
-            XContentParser.Token token;
-            String fieldName = null;
-            double discount = DEFAULT_BACKOFF_DISCOUNT;
-            while ((token = parser.nextToken()) != Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    fieldName = parser.currentName();
-                }
-                if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) {
-                    discount = parser.doubleValue();
-                }
-            }
-            return new StupidBackoff(discount);
-        }
-
-        @Override
-        public WordScorerFactory buildWordScorerFactory() {
-            return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-                    -> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount);
-        }
-    }
-
-    /**
-     * An <a href="http://en.wikipedia.org/wiki/Additive_smoothing">additive
-     * smoothing</a> model.
-     * <p>
-     * See <a
-     * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
-     * Smoothing</a> for details.
-     * </p>
-     */
-    public static final class Laplace extends SmoothingModel {
-        private double alpha = DEFAULT_LAPLACE_ALPHA;
-        private static final String NAME = "laplace";
-        private static final ParseField ALPHA_FIELD = new ParseField("alpha");
-        private static final ParseField PARSE_FIELD = new ParseField(NAME);
-        /**
-         * Default alpha parameter for laplace smoothing
-         */
-        public static final double DEFAULT_LAPLACE_ALPHA = 0.5;
-        public static final Laplace PROTOTYPE = new Laplace(DEFAULT_LAPLACE_ALPHA);
-
-        /**
-         * Creates a Laplace smoothing model.
-         *
-         */
-        public Laplace(double alpha) {
-            this.alpha = alpha;
-        }
-
-        /**
-         * @return the laplace model alpha parameter
-         */
-        public double getAlpha() {
-            return this.alpha;
-        }
-
-        @Override
-        protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(ALPHA_FIELD.getPreferredName(), alpha);
-            return builder;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeDouble(alpha);
-        }
-
-        @Override
-        public SmoothingModel readFrom(StreamInput in) throws IOException {
-            return new Laplace(in.readDouble());
-        }
-
-        @Override
-        protected boolean doEquals(SmoothingModel other) {
-            Laplace otherModel = (Laplace) other;
-            return Objects.equals(alpha, otherModel.alpha);
-        }
-
-        @Override
-        protected final int doHashCode() {
-            return Objects.hash(alpha);
-        }
-
-        @Override
-        public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
-            XContentParser parser = parseContext.parser();
-            XContentParser.Token token;
-            String fieldName = null;
-            double alpha = DEFAULT_LAPLACE_ALPHA;
-            while ((token = parser.nextToken()) != Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    fieldName = parser.currentName();
-                }
-                if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, ALPHA_FIELD)) {
-                    alpha = parser.doubleValue();
-                }
-            }
-            return new Laplace(alpha);
-        }
-
-        @Override
-        public WordScorerFactory buildWordScorerFactory() {
-            return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
-                    -> new LaplaceScorer(reader, terms,  field, realWordLikelyhood, separator, alpha);
-        }
-    }
-
-
-    public static abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent {
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(getWriteableName());
-            innerToXContent(builder,params);
-            builder.endObject();
-            return builder;
-        }
-
-        @Override
-        public final boolean equals(Object obj) {
-            if (this == obj) {
-                return true;
-            }
-            if (obj == null || getClass() != obj.getClass()) {
-                return false;
-            }
-            SmoothingModel other = (SmoothingModel) obj;
-            return doEquals(other);
-        }
-
-        public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
-            XContentParser parser = parseContext.parser();
-            ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
-            XContentParser.Token token;
-            String fieldName = null;
-            SmoothingModel model = null;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    fieldName = parser.currentName();
-                } else if (token == XContentParser.Token.START_OBJECT) {
-                    if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) {
-                        model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext);
-                    } else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) {
-                        model = Laplace.PROTOTYPE.innerFromXContent(parseContext);
-                    } else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) {
-                        model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext);
-                    } else {
-                        throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
-                    }
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(),
-                            "[smoothing] unknown token [" + token + "] after [" + fieldName + "]");
-                }
-            }
-            return model;
-        }
-
-        public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException;
-
-        @Override
-        public final int hashCode() {
-            /*
-             * Override hashCode here and forward to an abstract method to force extensions of this class to override hashCode in the same
-             * way that we force them to override equals. This also prevents false positives in CheckStyle's EqualsHashCode check.
-             */
-            return doHashCode();
-        }
-
-        public abstract WordScorerFactory buildWordScorerFactory();
-
-        /**
-         * subtype specific implementation of "equals".
-         */
-        protected abstract boolean doEquals(SmoothingModel other);
-
-        protected abstract int doHashCode();
-
-        protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
-    }
-
-    /**
-     * Linear interpolation smoothing model.
-     * <p>
-     * See <a
-     * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
-     * Smoothing</a> for details.
-     * </p>
-     */
-    public static final class LinearInterpolation extends SmoothingModel {
-        private static final String NAME = "linear";
-        public static final LinearInterpolation PROTOTYPE = new LinearInterpolation(0.8, 0.1, 0.1);
-        private final double trigramLambda;
-        private final double bigramLambda;
-        private final double unigramLambda;
-        private static final ParseField PARSE_FIELD = new ParseField(NAME);
-        private static final ParseField TRIGRAM_FIELD = new ParseField("trigram_lambda");
-        private static final ParseField BIGRAM_FIELD = new ParseField("bigram_lambda");
-        private static final ParseField UNIGRAM_FIELD = new ParseField("unigram_lambda");
-
-        /**
-         * Creates a linear interpolation smoothing model.
-         *
-         * Note: the lambdas must sum up to one.
-         *
-         * @param trigramLambda
-         *            the trigram lambda
-         * @param bigramLambda
-         *            the bigram lambda
-         * @param unigramLambda
-         *            the unigram lambda
-         */
-        public LinearInterpolation(double trigramLambda, double bigramLambda, double unigramLambda) {
-            double sum = trigramLambda + bigramLambda + unigramLambda;
-            if (Math.abs(sum - 1.0) > 0.001) {
-                throw new IllegalArgumentException("linear smoothing lambdas must sum to 1");
-            }
-            this.trigramLambda = trigramLambda;
-            this.bigramLambda = bigramLambda;
-            this.unigramLambda = unigramLambda;
-        }
-
-        public double getTrigramLambda() {
-            return this.trigramLambda;
-        }
-
-        public double getBigramLambda() {
-            return this.bigramLambda;
-        }
-
-        public double getUnigramLambda() {
-            return this.unigramLambda;
-        }
-
-        @Override
-        protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(TRIGRAM_FIELD.getPreferredName(), trigramLambda);
-            builder.field(BIGRAM_FIELD.getPreferredName(), bigramLambda);
-            builder.field(UNIGRAM_FIELD.getPreferredName(), unigramLambda);
-            return builder;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeDouble(trigramLambda);
-            out.writeDouble(bigramLambda);
-            out.writeDouble(unigramLambda);
-        }
-
-        @Override
-        public LinearInterpolation readFrom(StreamInput in) throws IOException {
-            return new LinearInterpolation(in.readDouble(), in.readDouble(), in.readDouble());
-        }
-
-        @Override
-        protected boolean doEquals(SmoothingModel other) {
-            final LinearInterpolation otherModel = (LinearInterpolation) other;
-            return Objects.equals(trigramLambda, otherModel.trigramLambda) &&
-                    Objects.equals(bigramLambda, otherModel.bigramLambda) &&
-                    Objects.equals(unigramLambda, otherModel.unigramLambda);
-        }
-
-        @Override
-        protected final int doHashCode() {
-            return Objects.hash(trigramLambda, bigramLambda, unigramLambda);
-        }
-
-        @Override
-        public LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException {
-            XContentParser parser = parseContext.parser();
-            XContentParser.Token token;
-            String fieldName = null;
-            double trigramLambda = 0.0;
-            double bigramLambda = 0.0;
-            double unigramLambda = 0.0;
-            ParseFieldMatcher matcher = parseContext.parseFieldMatcher();
-            while ((token = parser.nextToken()) != Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    fieldName = parser.currentName();
-                } else if (token.isValue()) {
-                    if (matcher.match(fieldName, TRIGRAM_FIELD)) {
-                        trigramLambda = parser.doubleValue();
-                        if (trigramLambda < 0) {
-                            throw new IllegalArgumentException("trigram_lambda must be positive");
-                        }
-                    } else if (matcher.match(fieldName, BIGRAM_FIELD)) {
-                        bigramLambda = parser.doubleValue();
-                        if (bigramLambda < 0) {
-                            throw new IllegalArgumentException("bigram_lambda must be positive");
-                        }
-                    } else if (matcher.match(fieldName, UNIGRAM_FIELD)) {
-                        unigramLambda = parser.doubleValue();
-                        if (unigramLambda < 0) {
-                            throw new IllegalArgumentException("unigram_lambda must be positive");
-                        }
-                    } else {
-                        throw new IllegalArgumentException(
-                                "suggester[phrase][smoothing][linear] doesn't support field [" + fieldName + "]");
-                    }
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(),
-                            "[" + NAME + "] unknown token [" + token + "] after [" + fieldName + "]");
-                }
-            }
-            return new LinearInterpolation(trigramLambda, bigramLambda, unigramLambda);
-        }
-
-        @Override
-        public WordScorerFactory buildWordScorerFactory() {
-            return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator) ->
-                        new LinearInterpoatingScorer(reader, terms, field, realWordLikelyhood, separator, trigramLambda, bigramLambda,
-                            unigramLambda);
-        }
-    }
-
     @Override
     protected PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext, String suggestionName) throws IOException {
         XContentParser parser = parseContext.parser();
@@ -873,7 +474,6 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
                                         "suggester[phrase][collate] query already set, doesn't support additional [" + fieldName + "]");
                             }
                             Template template = Template.parse(parser, parseFieldMatcher);
-                            // TODO remember to compile script in build() method
                             suggestion.collateQuery(template);
                         } else if (parseFieldMatcher.match(fieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS)) {
                             suggestion.collateParams(parser.map());
@@ -898,6 +498,98 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
         return suggestion;
     }
 
+
+    @Override
+    public SuggestionContext build(QueryShardContext context) throws IOException {
+        PhraseSuggestionContext suggestionContext = new PhraseSuggestionContext(context);
+        MapperService mapperService = context.getMapperService();
+        populateCommonFields(mapperService, suggestionContext);
+
+        suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator));
+        suggestionContext.setRealWordErrorLikelihood(this.realWordErrorLikelihood);
+        suggestionContext.setConfidence(this.confidence);
+        suggestionContext.setMaxErrors(this.maxErrors);
+        suggestionContext.setSeparator(BytesRefs.toBytesRef(this.separator));
+        suggestionContext.setRequireUnigram(this.forceUnigrams);
+        suggestionContext.setTokenLimit(this.tokenLimit);
+        suggestionContext.setPreTag(BytesRefs.toBytesRef(this.preTag));
+        suggestionContext.setPostTag(BytesRefs.toBytesRef(this.postTag));
+
+        if (this.gramSize != null) {
+            suggestionContext.setGramSize(this.gramSize);
+        }
+
+        for (List<CandidateGenerator> candidateGenerators : this.generators.values()) {
+            for (CandidateGenerator candidateGenerator : candidateGenerators) {
+                suggestionContext.addGenerator(candidateGenerator.build(mapperService));
+            }
+        }
+
+        if (this.model != null) {
+            suggestionContext.setModel(this.model.buildWordScorerFactory());
+        }
+
+        if (this.collateQuery != null) {
+            CompiledScript compiledScript = context.getScriptService().compile(this.collateQuery, ScriptContext.Standard.SEARCH,
+                    Collections.emptyMap());
+            suggestionContext.setCollateQueryScript(compiledScript);
+            if (this.collateParams != null) {
+                suggestionContext.setCollateScriptParams(this.collateParams);
+            }
+            suggestionContext.setCollatePrune(this.collatePrune);
+        }
+
+        // TODO remove this when field is mandatory in builder ctor
+        if (suggestionContext.getField() == null) {
+            throw new IllegalArgumentException("The required field option is missing");
+        }
+
+        MappedFieldType fieldType = mapperService.fullName(suggestionContext.getField());
+        if (fieldType == null) {
+            throw new IllegalArgumentException("No mapping found for field [" + suggestionContext.getField() + "]");
+        } else if (suggestionContext.getAnalyzer() == null) {
+            // no analyzer name passed in, so try the field's analyzer, or the default analyzer
+            if (fieldType.searchAnalyzer() == null) {
+                suggestionContext.setAnalyzer(mapperService.searchAnalyzer());
+            } else {
+                suggestionContext.setAnalyzer(fieldType.searchAnalyzer());
+            }
+        }
+
+        if (suggestionContext.model() == null) {
+            suggestionContext.setModel(StupidBackoffScorer.FACTORY);
+        }
+
+        if (this.gramSize == null || suggestionContext.generators().isEmpty()) {
+            final ShingleTokenFilterFactory.Factory shingleFilterFactory = SuggestUtils
+                    .getShingleFilterFactory(suggestionContext.getAnalyzer());
+            if (this.gramSize == null) {
+                // try to detect the shingle size
+                if (shingleFilterFactory != null) {
+                    suggestionContext.setGramSize(shingleFilterFactory.getMaxShingleSize());
+                    if (suggestionContext.getAnalyzer() == null && shingleFilterFactory.getMinShingleSize() > 1
+                            && !shingleFilterFactory.getOutputUnigrams()) {
+                        throw new IllegalArgumentException("The default analyzer for field: [" + suggestionContext.getField()
+                                + "] doesn't emit unigrams. If this is intentional try to set the analyzer explicitly");
+                    }
+                }
+            }
+            if (suggestionContext.generators().isEmpty()) {
+                if (shingleFilterFactory != null && shingleFilterFactory.getMinShingleSize() > 1
+                        && !shingleFilterFactory.getOutputUnigrams() && suggestionContext.getRequireUnigram()) {
+                    throw new IllegalArgumentException("The default candidate generator for phrase suggest can't operate on field: ["
+                            + suggestionContext.getField() + "] since it doesn't emit unigrams. "
+                            + "If this is intentional try to set the candidate generator field explicitly");
+                }
+                // use a default generator on the same field
+                DirectCandidateGenerator generator = new DirectCandidateGenerator();
+                generator.setField(suggestionContext.getField());
+                suggestionContext.addGenerator(generator);
+            }
+        }
+        return suggestionContext;
+    }
+
     private static void ensureNoSmoothing(PhraseSuggestionBuilder suggestion) {
         if (suggestion.smoothingModel() != null) {
             throw new IllegalArgumentException("only one smoothing model supported");
@@ -1010,5 +702,7 @@ public final class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSugge
         String getType();
 
         CandidateGenerator fromXContent(QueryParseContext parseContext) throws IOException;
+
+        PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException;
     }
 }

core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java

@@ -20,9 +20,9 @@ package org.elasticsearch.search.suggest.phrase;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
-import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
 import java.util.ArrayList;
@@ -54,8 +54,8 @@ class PhraseSuggestionContext extends SuggestionContext {
     private Map<String, Object> collateScriptParams = new HashMap<>(1);
     private WordScorer.WordScorerFactory scorer;
 
-    public PhraseSuggestionContext(Suggester<? extends PhraseSuggestionContext> suggester) {
+    public PhraseSuggestionContext(QueryShardContext shardContext) {
-        super(suggester);
+        super(PhraseSuggester.PROTOTYPE, shardContext);
     }
 
     public float maxErrors() {
@@ -154,8 +154,6 @@ class PhraseSuggestionContext extends SuggestionContext {
         public void postFilter(Analyzer postFilter) {
             this.postFilter = postFilter;
         }
-
-
     }
 
     public void setRequireUnigram(boolean requireUnigram) {
@@ -213,5 +211,4 @@ class PhraseSuggestionContext extends SuggestionContext {
     boolean collatePrune() {
         return prune;
     }
-
 }

core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.phrase;
+
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
+
+import java.io.IOException;
+
+public abstract class SmoothingModel implements NamedWriteable<SmoothingModel>, ToXContent {
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(getWriteableName());
+        innerToXContent(builder,params);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public final boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        SmoothingModel other = (SmoothingModel) obj;
+        return doEquals(other);
+    }
+
+    @Override
+    public final int hashCode() {
+        /*
+         * Override hashCode here and forward to an abstract method to force
+         * extensions of this class to override hashCode in the same way that we
+         * force them to override equals. This also prevents false positives in
+         * CheckStyle's EqualsHashCode check.
+         */
+        return doHashCode();
+    }
+
+    protected abstract int doHashCode();
+
+    public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        ParseFieldMatcher parseFieldMatcher = parseContext.parseFieldMatcher();
+        XContentParser.Token token;
+        String fieldName = null;
+        SmoothingModel model = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) {
+                    model = LinearInterpolation.PROTOTYPE.innerFromXContent(parseContext);
+                } else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) {
+                    model = Laplace.PROTOTYPE.innerFromXContent(parseContext);
+                } else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) {
+                    model = StupidBackoff.PROTOTYPE.innerFromXContent(parseContext);
+                } else {
+                    throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
+                }
+            } else {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "[smoothing] unknown token [" + token + "] after [" + fieldName + "]");
+            }
+        }
+        return model;
+    }
+
+    public abstract SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException;
+
+    public abstract WordScorerFactory buildWordScorerFactory();
+
+    /**
+     * subtype specific implementation of "equals".
+     */
+    protected abstract boolean doEquals(SmoothingModel other);
+
+    protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
+}

core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java

@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest.phrase;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * A "stupid-backoff" smoothing model simialr to <a
+ * href="http://en.wikipedia.org/wiki/Katz's_back-off_model"> Katz's
+ * Backoff</a>. This model is used as the default if no model is configured.
+ * <p>
+ * See <a
+ * href="http://en.wikipedia.org/wiki/N-gram#Smoothing_techniques">N-Gram
+ * Smoothing</a> for details.
+ * </p>
+ */
+public final class StupidBackoff extends SmoothingModel {
+    /**
+     * Default discount parameter for {@link StupidBackoff} smoothing
+     */
+    public static final double DEFAULT_BACKOFF_DISCOUNT = 0.4;
+    public static final StupidBackoff PROTOTYPE = new StupidBackoff(DEFAULT_BACKOFF_DISCOUNT);
+    private double discount = DEFAULT_BACKOFF_DISCOUNT;
+    private static final String NAME = "stupid_backoff";
+    private static final ParseField DISCOUNT_FIELD = new ParseField("discount");
+    static final ParseField PARSE_FIELD = new ParseField(NAME);
+
+    /**
+     * Creates a Stupid-Backoff smoothing model.
+     *
+     * @param discount
+     *            the discount given to lower order ngrams if the higher order ngram doesn't exits
+     */
+    public StupidBackoff(double discount) {
+        this.discount = discount;
+    }
+
+    /**
+     * @return the discount parameter of the model
+     */
+    public double getDiscount() {
+        return this.discount;
+    }
+
+    @Override
+    protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(DISCOUNT_FIELD.getPreferredName(), discount);
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeDouble(discount);
+    }
+
+    @Override
+    public StupidBackoff readFrom(StreamInput in) throws IOException {
+        return new StupidBackoff(in.readDouble());
+    }
+
+    @Override
+    protected boolean doEquals(SmoothingModel other) {
+        StupidBackoff otherModel = (StupidBackoff) other;
+        return Objects.equals(discount, otherModel.discount);
+    }
+
+    @Override
+    protected final int doHashCode() {
+        return Objects.hash(discount);
+    }
+
+    @Override
+    public SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser parser = parseContext.parser();
+        XContentParser.Token token;
+        String fieldName = null;
+        double discount = DEFAULT_BACKOFF_DISCOUNT;
+        while ((token = parser.nextToken()) != Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            }
+            if (token.isValue() && parseContext.parseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) {
+                discount = parser.doubleValue();
+            }
+        }
+        return new StupidBackoff(discount);
+    }
+
+    @Override
+    public WordScorerFactory buildWordScorerFactory() {
+        return (IndexReader reader, Terms terms, String field, double realWordLikelyhood, BytesRef separator)
+                -> new StupidBackoffScorer(reader, terms, field, realWordLikelyhood, separator, discount);
+    }
+}

core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestParser.java

@@ -20,8 +20,8 @@ package org.elasticsearch.search.suggest.term;
 
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
 import org.elasticsearch.search.suggest.SuggestContextParser;
 import org.elasticsearch.search.suggest.SuggestUtils;
@@ -38,10 +38,11 @@ public final class TermSuggestParser implements SuggestContextParser {
     }
 
     @Override
-    public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService, IndexFieldDataService fieldDataService) throws IOException {
+    public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, QueryShardContext shardContext) throws IOException {
+        MapperService mapperService = shardContext.getMapperService();
         XContentParser.Token token;
         String fieldName = null;
-        TermSuggestionContext suggestion = new TermSuggestionContext(suggester);
+        TermSuggestionContext suggestion = new TermSuggestionContext(shardContext);
         DirectSpellcheckerSettings settings = suggestion.getDirectSpellCheckerSettings();
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {

core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java

@@ -40,6 +40,8 @@ import java.util.List;
 
 public final class TermSuggester extends Suggester<TermSuggestionContext> {
 
+    public static final TermSuggester PROTOTYPE = new TermSuggester();
+
     @Override
     public TermSuggestion innerExecute(String name, TermSuggestionContext suggestion, IndexSearcher searcher, CharsRefBuilder spare)
             throws IOException {

core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java

@@ -26,7 +26,9 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -381,6 +383,12 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
         return suggestion;
     }
 
+    @Override
+    protected SuggestionContext build(QueryShardContext context) throws IOException {
+        // NORELEASE
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public String getWriteableName() {
         return SUGGESTION_NAME;

core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java

@@ -18,20 +18,19 @@
  */
 package org.elasticsearch.search.suggest.term;
 
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
-import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
 final class TermSuggestionContext extends SuggestionContext {
 
     private final DirectSpellcheckerSettings settings = new DirectSpellcheckerSettings();
 
-    public TermSuggestionContext(Suggester<? extends TermSuggestionContext> suggester) {
+    public TermSuggestionContext(QueryShardContext shardContext) {
-        super(suggester);
+        super(TermSuggester.PROTOTYPE, shardContext);
     }
 
     public DirectSpellcheckerSettings getDirectSpellCheckerSettings() {
         return settings;
     }
-
 }

core/src/test/java/org/elasticsearch/index/IndexModuleTests.java

@@ -63,8 +63,8 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.engine.MockEngineFactory;

core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.suggest;
 
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
@@ -31,19 +32,48 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperBuilders;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.core.StringFieldMapper;
+import org.elasticsearch.index.mapper.core.StringFieldMapper.StringFieldType;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptContextRegistry;
+import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptServiceTests.TestEngineService;
+import org.elasticsearch.script.ScriptSettings;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.watcher.ResourceWatcherService;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
@@ -51,13 +81,35 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
 
     private static final int NUMBER_OF_TESTBUILDERS = 20;
     protected static NamedWriteableRegistry namedWriteableRegistry;
-    private static final Suggesters suggesters = new Suggesters(Collections.emptyMap(), null, null);
+    private static Suggesters suggesters;
+    private static ScriptService scriptService;
+    private static SuggestParseElement parseElement;
 
     /**
      * setup for the whole base test class
      */
     @BeforeClass
-    public static void init() {
+    public static void init() throws IOException {
+        Path genericConfigFolder = createTempDir();
+        Settings baseSettings = settingsBuilder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .put(Environment.PATH_CONF_SETTING.getKey(), genericConfigFolder)
+                .build();
+        Environment environment = new Environment(baseSettings);
+        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry
+                .ScriptEngineRegistration(TestEngineService.class, TestEngineService.TYPES)));
+        ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+        scriptService = new ScriptService(baseSettings, environment, Collections.singleton(new TestEngineService()),
+                new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
+            @Override
+            public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params) {
+                return new CompiledScript(ScriptType.INLINE, "mockName", "mocklang", script);
+            }
+        };
+        suggesters = new Suggesters(Collections.emptyMap());
+        parseElement = new SuggestParseElement(suggesters);
+
         namedWriteableRegistry = new NamedWriteableRegistry();
         namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, TermSuggestionBuilder.PROTOTYPE);
         namedWriteableRegistry.registerPrototype(SuggestionBuilder.class, PhraseSuggestionBuilder.PROTOTYPE);
@@ -69,7 +121,6 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
         namedWriteableRegistry = null;
     }
 
-
     /**
      * Test serialization and deserialization of the suggestion builder
      */
@@ -88,13 +139,13 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
      */
     protected SB randomTestBuilder() {
         SB randomSuggestion = randomSuggestionBuilder();
+        randomSuggestion.field(randomAsciiOfLengthBetween(2, 20));
         maybeSet(randomSuggestion::text, randomAsciiOfLengthBetween(2, 20));
         maybeSet(randomSuggestion::prefix, randomAsciiOfLengthBetween(2, 20));
         maybeSet(randomSuggestion::regex, randomAsciiOfLengthBetween(2, 20));
-        maybeSet(randomSuggestion::field, randomAsciiOfLengthBetween(2, 20));
         maybeSet(randomSuggestion::analyzer, randomAsciiOfLengthBetween(2, 20));
         maybeSet(randomSuggestion::size, randomIntBetween(1, 20));
-        maybeSet(randomSuggestion::shardSize, randomInt(20));
+        maybeSet(randomSuggestion::shardSize, randomIntBetween(1, 20));
         return randomSuggestion;
     }
 
@@ -137,7 +188,8 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
     }
 
     /**
-     *  creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
+     * creates random suggestion builder, renders it to xContent and back to new
+     * instance that should be equal to original
      */
     public void testFromXContent() throws IOException {
         QueryParseContext context = new QueryParseContext(null);
@@ -166,6 +218,97 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
         }
     }
 
+    /**
+     * parses random suggestion builder via old parseElement method and via
+     * build, comparing the results for equality
+     */
+    public void testBuild() throws IOException {
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), Settings.EMPTY);
+
+        AnalysisService mockAnalysisService = new AnalysisService(idxSettings, Collections.emptyMap(), Collections.emptyMap(),
+                Collections.emptyMap(), Collections.emptyMap()) {
+            @Override
+            public NamedAnalyzer analyzer(String name) {
+                return new NamedAnalyzer(name, new WhitespaceAnalyzer());
+            }
+        };
+
+        MapperService mockMapperService = new MapperService(idxSettings, mockAnalysisService, null, new IndicesModule().getMapperRegistry(),
+                null) {
+            @Override
+            public MappedFieldType fullName(String fullName) {
+                return new StringFieldType();
+            }
+        };
+
+        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, mockMapperService, null, scriptService,
+                null) {
+            @Override
+            public MappedFieldType fieldMapper(String name) {
+                StringFieldMapper.Builder builder = MapperBuilders.stringField(name);
+                return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType();
+            }
+        };
+        mockShardContext.setMapUnmappedFieldAsString(true);
+
+        for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
+            SuggestBuilder suggestBuilder = new SuggestBuilder();
+            SB suggestionBuilder = randomTestBuilder();
+            suggestBuilder.addSuggestion(suggestionBuilder);
+
+            if (suggestionBuilder.text() == null) {
+                // we either need suggestion text or global text
+                suggestBuilder.setText(randomAsciiOfLengthBetween(5, 50));
+            }
+            if (suggestionBuilder.text() != null && suggestionBuilder.prefix() != null) {
+                suggestionBuilder.prefix(null);
+            }
+
+            XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+            if (randomBoolean()) {
+                xContentBuilder.prettyPrint();
+            }
+            suggestBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+
+            XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes());
+            parser.nextToken(); // set cursor to START_OBJECT
+            SuggestionSearchContext parsedSuggestionSearchContext = parseElement.parseInternal(parser, mockShardContext);
+
+            SuggestionSearchContext buildSuggestSearchContext = suggestBuilder.build(mockShardContext);
+            assertEquals(parsedSuggestionSearchContext.suggestions().size(), buildSuggestSearchContext.suggestions().size());
+            Iterator<Entry<String, SuggestionContext>> iterator = buildSuggestSearchContext.suggestions().entrySet().iterator();
+            for (Entry<String, SuggestionContext> entry : parsedSuggestionSearchContext.suggestions().entrySet()) {
+                Entry<String, SuggestionContext> other = iterator.next();
+                assertEquals(entry.getKey(), other.getKey());
+
+                SuggestionContext oldSchoolContext = entry.getValue();
+                SuggestionContext newSchoolContext = other.getValue();
+                assertNotSame(oldSchoolContext, newSchoolContext);
+                // deep comparison of analyzers is difficult here, but we check they are set or not set
+                if (oldSchoolContext.getAnalyzer() != null) {
+                    assertNotNull(newSchoolContext.getAnalyzer());
+                } else {
+                    assertNull(newSchoolContext.getAnalyzer());
+                }
+                assertEquals(oldSchoolContext.getField(), newSchoolContext.getField());
+                assertEquals(oldSchoolContext.getPrefix(), newSchoolContext.getPrefix());
+                assertEquals(oldSchoolContext.getRegex(), newSchoolContext.getRegex());
+                assertEquals(oldSchoolContext.getShardSize(), newSchoolContext.getShardSize());
+                assertEquals(oldSchoolContext.getSize(), newSchoolContext.getSize());
+                assertEquals(oldSchoolContext.getSuggester().getClass(), newSchoolContext.getSuggester().getClass());
+                assertEquals(oldSchoolContext.getText(), newSchoolContext.getText());
+                assertEquals(oldSchoolContext.getClass(), newSchoolContext.getClass());
+
+                assertSuggestionContext(oldSchoolContext, newSchoolContext);
+            }
+        }
+    }
+
+    /**
+     * compare two SuggestionContexte implementations for the special suggestion type under test
+     */
+    protected abstract void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion);
+
     private SB mutate(SB firstBuilder) throws IOException {
         SB mutation = serializedCopy(firstBuilder);
         assertNotSame(mutation, firstBuilder);
@@ -201,14 +344,16 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
     }
 
     /**
-     * take and input {@link SuggestBuilder} and return another one that is different in one aspect (to test non-equality)
+     * take and input {@link SuggestBuilder} and return another one that is
+     * different in one aspect (to test non-equality)
      */
     protected abstract void mutateSpecificParameters(SB firstBuilder) throws IOException;
 
     @SuppressWarnings("unchecked")
     protected SB serializedCopy(SB original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
-            output.writeSuggestion(original);;
+            output.writeSuggestion(original);
+            ;
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
                 return (SB) in.readSuggestion();
             }
@@ -222,7 +367,8 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
     }
 
     /**
-     * helper to get a random value in a certain range that's different from the input
+     * helper to get a random value in a certain range that's different from the
+     * input
      */
     protected static <T> T randomValueOtherThan(T input, Supplier<T> randomSupplier) {
         T randomValue = null;

core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java

@@ -21,6 +21,7 @@ package org.elasticsearch.search.suggest;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -54,9 +55,9 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions
 
     @Override
     public SuggestContextParser getContextParser() {
-        return (parser, mapperService, fieldData) -> {
+        return (parser, shardContext) -> {
             Map<String, Object> options = parser.map();
-            CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(CustomSuggester.this, options);
+            CustomSuggestionsContext suggestionContext = new CustomSuggestionsContext(shardContext, options);
             suggestionContext.setField((String) options.get("field"));
             return suggestionContext;
         };
@@ -66,8 +67,8 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions
 
         public Map<String, Object> options;
 
-        public CustomSuggestionsContext(Suggester suggester, Map<String, Object> options) {
+        public CustomSuggestionsContext(QueryShardContext context, Map<String, Object> options) {
-            super(suggester);
+            super(new CustomSuggester(), context);
             this.options = options;
         }
     }

core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java

@@ -25,7 +25,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -132,6 +134,12 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
             return new CustomSuggestionBuilder(name, randomField, randomSuffix);
         }
 
+        @Override
+        protected SuggestionContext build(QueryShardContext context) throws IOException {
+            // NORELEASE
+            return null;
+        }
+
     }
 
 }

core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java

@@ -32,12 +32,12 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
 import org.elasticsearch.search.suggest.completion.WritableTestCase;
+import org.elasticsearch.search.suggest.phrase.Laplace;
+import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilderTests;
+import org.elasticsearch.search.suggest.phrase.SmoothingModel;
+import org.elasticsearch.search.suggest.phrase.StupidBackoff;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 
 import java.io.IOException;
@@ -62,7 +62,7 @@ public class SuggestBuilderTests extends WritableTestCase<SuggestBuilder> {
      *  creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
      */
     public void testFromXContent() throws IOException {
-        Suggesters suggesters = new Suggesters(Collections.emptyMap(), null, null);
+        Suggesters suggesters = new Suggesters(Collections.emptyMap());
         QueryParseContext context = new QueryParseContext(null);
         context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
         for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {

core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java

@@ -34,15 +34,10 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.index.mapper.MapperBuilders;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.index.mapper.core.StringFieldMapper.StringFieldType;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
@@ -171,19 +166,10 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
             }
         };
 
-        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, mockMapperService, null, null, null) {
-            @Override
-            public MappedFieldType fieldMapper(String name) {
-                StringFieldMapper.Builder builder = MapperBuilders.stringField(name);
-                return builder.build(new Mapper.BuilderContext(idxSettings.getSettings(), new ContentPath(1))).fieldType();
-            }
-        };
-        mockShardContext.setMapUnmappedFieldAsString(true);
-
         for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {
             DirectCandidateGeneratorBuilder generator = randomCandidateGenerator();
             // first, build via DirectCandidateGenerator#build()
-            DirectCandidateGenerator contextGenerator = generator.build(mockShardContext);
+            DirectCandidateGenerator contextGenerator = generator.build(mockMapperService);
 
             // second, render random test generator to xContent and parse using
             // PhraseSuggestParser
@@ -195,28 +181,32 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
             XContentParser parser = XContentHelper.createParser(builder.bytes());
 
             DirectCandidateGenerator secondGenerator = PhraseSuggestParser.parseCandidateGenerator(parser,
-                    mockShardContext.getMapperService(), mockShardContext.parseFieldMatcher());
+                    mockMapperService, ParseFieldMatcher.EMPTY);
 
             // compare their properties
             assertNotSame(contextGenerator, secondGenerator);
-            assertEquals(contextGenerator.field(), secondGenerator.field());
+            assertEqualGenerators(contextGenerator, secondGenerator);
-            assertEquals(contextGenerator.accuracy(), secondGenerator.accuracy(), Float.MIN_VALUE);
-            assertEquals(contextGenerator.maxTermFreq(), secondGenerator.maxTermFreq(), Float.MIN_VALUE);
-            assertEquals(contextGenerator.maxEdits(), secondGenerator.maxEdits());
-            assertEquals(contextGenerator.maxInspections(), secondGenerator.maxInspections());
-            assertEquals(contextGenerator.minDocFreq(), secondGenerator.minDocFreq(), Float.MIN_VALUE);
-            assertEquals(contextGenerator.minWordLength(), secondGenerator.minWordLength());
-            assertEquals(contextGenerator.postFilter(), secondGenerator.postFilter());
-            assertEquals(contextGenerator.prefixLength(), secondGenerator.prefixLength());
-            assertEquals(contextGenerator.preFilter(), secondGenerator.preFilter());
-            assertEquals(contextGenerator.sort(), secondGenerator.sort());
-            assertEquals(contextGenerator.size(), secondGenerator.size());
-            // some instances of StringDistance don't support equals, just checking the class here
-            assertEquals(contextGenerator.stringDistance().getClass(), secondGenerator.stringDistance().getClass());
-            assertEquals(contextGenerator.suggestMode(), secondGenerator.suggestMode());
         }
     }
 
+    public static void assertEqualGenerators(DirectCandidateGenerator first, DirectCandidateGenerator second) {
+        assertEquals(first.field(), second.field());
+        assertEquals(first.accuracy(), second.accuracy(), Float.MIN_VALUE);
+        assertEquals(first.maxTermFreq(), second.maxTermFreq(), Float.MIN_VALUE);
+        assertEquals(first.maxEdits(), second.maxEdits());
+        assertEquals(first.maxInspections(), second.maxInspections());
+        assertEquals(first.minDocFreq(), second.minDocFreq(), Float.MIN_VALUE);
+        assertEquals(first.minWordLength(), second.minWordLength());
+        assertEquals(first.postFilter(), second.postFilter());
+        assertEquals(first.prefixLength(), second.prefixLength());
+        assertEquals(first.preFilter(), second.preFilter());
+        assertEquals(first.sort(), second.sort());
+        assertEquals(first.size(), second.size());
+        // some instances of StringDistance don't support equals, just checking the class here
+        assertEquals(first.stringDistance().getClass(), second.stringDistance().getClass());
+        assertEquals(first.suggestMode(), second.suggestMode());
+    }
+
     /**
      * test that bad xContent throws exception
      */

core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java

@@ -19,9 +19,6 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
-
 import static org.hamcrest.Matchers.instanceOf;
 
 public class LaplaceModelTests extends SmoothingModelTestCase {

core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java

@@ -19,9 +19,6 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
-
 import static org.hamcrest.Matchers.instanceOf;
 
 public class LinearInterpolationModelTests extends SmoothingModelTestCase {

core/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java

@@ -21,16 +21,17 @@ package org.elasticsearch.search.suggest.phrase;
 
 import org.elasticsearch.script.Template;
 import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.instanceOf;
+
 public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<PhraseSuggestionBuilder> {
 
     @BeforeClass
@@ -70,7 +71,7 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
         }
         maybeSet(testBuilder::gramSize, randomIntBetween(1, 5));
         maybeSet(testBuilder::forceUnigrams, randomBoolean());
-        maybeSet(testBuilder::tokenLimit, randomInt(20));
+        maybeSet(testBuilder::tokenLimit, randomIntBetween(1, 20));
         if (randomBoolean()) {
             testBuilder.smoothingModel(randomSmoothingModel());
         }
@@ -115,7 +116,7 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
             builder.gramSize(randomValueOtherThan(builder.gramSize(), () -> randomIntBetween(1, 5)));
             break;
         case 4:
-            builder.tokenLimit(randomValueOtherThan(builder.tokenLimit(), () -> randomInt(20)));
+            builder.tokenLimit(randomValueOtherThan(builder.tokenLimit(), () -> randomIntBetween(1, 20)));
             break;
         case 5:
             builder.separator(randomValueOtherThan(builder.separator(), () -> randomAsciiOfLengthBetween(1, 10)));
@@ -158,4 +159,38 @@ public class PhraseSuggestionBuilderTests extends AbstractSuggestionBuilderTestC
         }
     }
 
+    @Override
+    protected void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion) {
+        assertThat(oldSuggestion, instanceOf(PhraseSuggestionContext.class));
+        assertThat(newSuggestion, instanceOf(PhraseSuggestionContext.class));
+        PhraseSuggestionContext oldPhraseSuggestion = (PhraseSuggestionContext) oldSuggestion;
+        PhraseSuggestionContext newPhraseSuggestion = (PhraseSuggestionContext) newSuggestion;
+        assertEquals(oldPhraseSuggestion.confidence(), newPhraseSuggestion.confidence(), Float.MIN_VALUE);
+        assertEquals(oldPhraseSuggestion.collatePrune(), newPhraseSuggestion.collatePrune());
+        assertEquals(oldPhraseSuggestion.gramSize(), newPhraseSuggestion.gramSize());
+        assertEquals(oldPhraseSuggestion.realworldErrorLikelyhood(), newPhraseSuggestion.realworldErrorLikelyhood(), Float.MIN_VALUE);
+        assertEquals(oldPhraseSuggestion.maxErrors(), newPhraseSuggestion.maxErrors(), Float.MIN_VALUE);
+        assertEquals(oldPhraseSuggestion.separator(), newPhraseSuggestion.separator());
+        assertEquals(oldPhraseSuggestion.getTokenLimit(), newPhraseSuggestion.getTokenLimit());
+        assertEquals(oldPhraseSuggestion.getRequireUnigram(), newPhraseSuggestion.getRequireUnigram());
+        assertEquals(oldPhraseSuggestion.getPreTag(), newPhraseSuggestion.getPreTag());
+        assertEquals(oldPhraseSuggestion.getPostTag(), newPhraseSuggestion.getPostTag());
+        if (oldPhraseSuggestion.getCollateQueryScript() != null) {
+            // only assert that we have a compiled script on the other side
+            assertNotNull(newPhraseSuggestion.getCollateQueryScript());
+        }
+        if (oldPhraseSuggestion.generators() != null) {
+            assertNotNull(newPhraseSuggestion.generators());
+            assertEquals(oldPhraseSuggestion.generators().size(), newPhraseSuggestion.generators().size());
+            Iterator<DirectCandidateGenerator> secondList = newPhraseSuggestion.generators().iterator();
+            for (DirectCandidateGenerator candidateGenerator : newPhraseSuggestion.generators()) {
+                DirectCandidateGeneratorTests.assertEqualGenerators(candidateGenerator, secondList.next());
+            }
+        }
+        assertEquals(oldPhraseSuggestion.getCollateScriptParams(), newPhraseSuggestion.getCollateScriptParams());
+        if (oldPhraseSuggestion.model() != null) {
+            assertNotNull(newPhraseSuggestion.model());
+        }
+    }
+
 }

core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java

@@ -45,10 +45,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.Laplace;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.LinearInterpolation;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;

core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java

@@ -19,9 +19,6 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.SmoothingModel;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.StupidBackoff;
-
 import static org.hamcrest.Matchers.instanceOf;
 
 public class StupidBackoffModelTests extends SmoothingModelTestCase {

core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.search.suggest.term;
 
 import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
@@ -33,6 +34,14 @@ import static org.hamcrest.Matchers.notNullValue;
  */
 public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCase<TermSuggestionBuilder> {
 
+    /**
+     *  creates random suggestion builder, renders it to xContent and back to new instance that should be equal to original
+     */
+    @Override
+    public void testBuild() throws IOException {
+        // skip for now
+    }
+
     @Override
     protected TermSuggestionBuilder randomSuggestionBuilder() {
         TermSuggestionBuilder testBuilder = new TermSuggestionBuilder(randomAsciiOfLength(10));
@@ -245,4 +254,9 @@ public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCas
         assertThat(builder.suggestMode(), notNullValue());
     }
 
+    @Override
+    protected void assertSuggestionContext(SuggestionContext oldSuggestion, SuggestionContext newSuggestion) {
+        // put assertions on TermSuggestionContext here
+    }
+
 }

modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java

@@ -20,38 +20,6 @@
 package org.elasticsearch.messy.tests;
 
 
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion;
-import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
-import static org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.candidateGenerator;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
-import static org.hamcrest.Matchers.anyOf;
-import static org.hamcrest.Matchers.endsWith;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.nullValue;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -71,13 +39,47 @@ import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder;
+import org.elasticsearch.search.suggest.phrase.Laplace;
+import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.elasticsearch.search.suggest.phrase.StupidBackoff;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SortBy;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion;
+import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
+
 /**
  * Integration tests for term and phrase suggestions.  Many of these tests many requests that vary only slightly from one another.  Where
  * possible these tests should declare for the first request, make the request, modify the configuration for the next request, make that
@@ -227,6 +229,16 @@ public class SuggestSearchTests extends ESIntegTestCase {
         assertSuggestionSize(searchSuggest, 0, 0, "did_you_mean");
     }
 
+    /**
+     * Creates a new {@link DirectCandidateGeneratorBuilder}
+     *
+     * @param field
+     *            the field this candidate generator operates on.
+     */
+    private DirectCandidateGeneratorBuilder candidateGenerator(String field) {
+        return new DirectCandidateGeneratorBuilder(field);
+    }
+
     // see #2729
     public void testSizeOneShard() throws Exception {
         prepareCreate("test").setSettings(
@@ -286,7 +298,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
         refresh();
 
         PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("did_you_mean").field("name.shingled")
-                .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
+                .addCandidateGenerator(candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
                 .gramSize(3);
         Suggest searchSuggest = searchSuggest( "ice tea", phraseSuggestion);
         assertSuggestion(searchSuggest, 0, 0, "did_you_mean", "iced tea");
@@ -439,7 +451,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
 
         Suggest searchSuggest = searchSuggest( "a an the",
                 phraseSuggestion("simple_phrase").field("body").gramSize(1)
-                        .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always"))
+                        .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
                         .size(1));
         assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
     }
@@ -475,13 +487,13 @@ public class SuggestSearchTests extends ESIntegTestCase {
 
         Suggest searchSuggest = searchSuggest( "hello word",
                 phraseSuggestion("simple_phrase").field("body")
-                        .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always"))
+                        .addCandidateGenerator(candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always"))
                         .size(1).confidence(1.0f));
         assertSuggestion(searchSuggest, 0, "simple_phrase", "hello words");
 
         searchSuggest = searchSuggest( "hello word",
                 phraseSuggestion("simple_phrase").field("body")
-                        .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always"))
+                        .addCandidateGenerator(candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always"))
                         .size(1).confidence(1.0f));
         assertSuggestion(searchSuggest, 0, "simple_phrase", "hello world");
     }
@@ -573,17 +585,17 @@ public class SuggestSearchTests extends ESIntegTestCase {
         // set all mass to trigrams (not indexed)
         phraseSuggest.clearCandidateGenerators()
             .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
-            .smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(1,0,0));
+            .smoothingModel(new LinearInterpolation(1,0,0));
         searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
         assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
 
         // set all mass to bigrams
-        phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0,1,0));
+        phraseSuggest.smoothingModel(new LinearInterpolation(0,1,0));
         searchSuggest =  searchSuggest( "Xor the Got-Jewel", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
         // distribute mass
-        phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0.4,0.4,0.2));
+        phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
         searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
@@ -591,15 +603,15 @@ public class SuggestSearchTests extends ESIntegTestCase {
         assertSuggestion(searchSuggest, 0, "simple_phrase", "american ace");
 
         // try all smoothing methods
-        phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.LinearInterpolation(0.4,0.4,0.2));
+        phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
         searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
-        phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.Laplace(0.2));
+        phraseSuggest.smoothingModel(new Laplace(0.2));
         searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
-        phraseSuggest.smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1));
+        phraseSuggest.smoothingModel(new StupidBackoff(0.1));
         searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
 
@@ -608,7 +620,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
         searchSuggest = searchSuggest( "Xor the Got-Jewel", phraseSuggest);
         assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
 
-        phraseSuggest.tokenLimit(15).smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1));
+        phraseSuggest.tokenLimit(15).smoothingModel(new StupidBackoff(0.1));
         searchSuggest = searchSuggest( "Xor the Got-Jewel Xor the Got-Jewel Xor the Got-Jewel", phraseSuggest);
         assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel xorr the god jewel xorr the god jewel");
         // Check the name this time because we're repeating it which is funky
@@ -671,7 +683,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
                 .gramSize(2)
                 .analyzer("body")
                 .addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(1).accuracy(0.1f))
-                .smoothingModel(new PhraseSuggestionBuilder.StupidBackoff(0.1))
+                .smoothingModel(new StupidBackoff(0.1))
                 .maxErrors(1.0f)
                 .size(5);
         Suggest searchSuggest = searchSuggest( "Xorr the Gut-Jewel", phraseSuggestion);
@@ -931,7 +943,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
 
         Suggest searchSuggest = searchSuggest("nobel prize", phraseSuggestion("simple_phrase")
                 .field("body")
-                .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
+                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
                 .confidence(2f)
                 .maxErrors(5f)
                 .size(1));
@@ -939,7 +951,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
 
         searchSuggest = searchSuggest("noble prize", phraseSuggestion("simple_phrase")
                 .field("body")
-                .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
+                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
                 .confidence(2f)
                 .maxErrors(5f)
                 .size(1));
@@ -1070,7 +1082,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
 
         PhraseSuggestionBuilder suggest = phraseSuggestion("title")
                 .field("title")
-                .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title")
+                .addCandidateGenerator(candidateGenerator("title")
                         .suggestMode("always")
                         .maxTermFreq(.99f)
                         .size(1000) // Setting a silly high size helps of generate a larger list of candidates for testing.
@@ -1135,7 +1147,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
         // suggest without collate
         PhraseSuggestionBuilder suggest = phraseSuggestion("title")
                 .field("title")
-                .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("title")
+                .addCandidateGenerator(candidateGenerator("title")
                         .suggestMode("always")
                         .maxTermFreq(.99f)
                         .size(10)