Merge pull request #16599 from s1monw/add_rewrite_infra

Add infrastructure to rewrite query builders

core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -90,7 +90,6 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
                 logger.debug("failed to optimize search type, continue as normal", e);
             }
         }
-
         if (searchRequest.searchType() == DFS_QUERY_THEN_FETCH) {
             dfsQueryThenFetchAction.execute(searchRequest, listener);
         } else if (searchRequest.searchType() == SearchType.QUERY_THEN_FETCH) {

core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java

@@ -27,6 +27,8 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.indices.InvalidAliasNameException;
 
@@ -143,7 +145,9 @@ public class AliasValidator extends AbstractComponent {
     private void validateAliasFilter(XContentParser parser, QueryShardContext queryShardContext) throws IOException {
         try {
             queryShardContext.reset(parser);
-            queryShardContext.parseContext().parseInnerQueryBuilder().toFilter(queryShardContext);
+            QueryParseContext queryParseContext = queryShardContext.parseContext();
+            QueryBuilder<?> queryBuilder = QueryBuilder.rewriteQuery(queryParseContext.parseInnerQueryBuilder(), queryShardContext);
+            queryBuilder.toFilter(queryShardContext);
         } finally {
             queryShardContext.reset(null);
             parser.close();

core/src/main/java/org/elasticsearch/index/IndexService.java

@@ -420,7 +420,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
      * Creates a new QueryShardContext. The context has not types set yet, if types are required set them via {@link QueryShardContext#setTypes(String...)}
      */
     public QueryShardContext newQueryShardContext() {
-        return new QueryShardContext(indexSettings, nodeServicesProvider.getClient(), indexCache.bitsetFilterCache(), indexFieldData, mapperService(), similarityService(), nodeServicesProvider.getScriptService(), nodeServicesProvider.getIndicesQueriesRegistry());
+        return new QueryShardContext(indexSettings, indexCache.bitsetFilterCache(), indexFieldData, mapperService(), similarityService(), nodeServicesProvider.getScriptService(), nodeServicesProvider.getIndicesQueriesRegistry());
     }
 
     ThreadPool getThreadPool() {

core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java

@@ -258,4 +258,23 @@ public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>>
         }
         return queries;
     }
+
+    @Override
+    public final QueryBuilder<?> rewrite(QueryRewriteContext queryShardContext) throws IOException {
+        QueryBuilder rewritten = doRewrite(queryShardContext);
+        if (rewritten == this) {
+            return rewritten;
+        }
+        if (queryName() != null && rewritten.queryName() == null) { // we inherit the name
+            rewritten.queryName(queryName());
+        }
+        if (boost() != DEFAULT_BOOST && rewritten.boost() == DEFAULT_BOOST) {
+            rewritten.boost(boost());
+        }
+        return rewritten;
+    }
+
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+        return this;
+    }
 }

core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java

@@ -33,6 +33,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.common.lucene.search.Queries.fixNegativeQueryIfNeeded;
 
@@ -272,6 +273,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
         if (booleanQuery.clauses().isEmpty()) {
             return new MatchAllDocsQuery();
         }
+
         final String minimumShouldMatch;
         if (context.isFilter() && this.minimumShouldMatch == null && shouldClauses.size() > 0) {
             minimumShouldMatch = "1";
@@ -346,4 +348,40 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
         out.writeBoolean(disableCoord);
         out.writeOptionalString(minimumShouldMatch);
     }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        BoolQueryBuilder newBuilder = new BoolQueryBuilder();
+        boolean changed = false;
+        final int clauses = mustClauses.size() + mustNotClauses.size() + filterClauses.size() + shouldClauses.size();
+        if (clauses == 0) {
+            return new MatchAllQueryBuilder().boost(boost()).queryName(queryName());
+        }
+        changed |= rewriteClauses(queryRewriteContext, mustClauses, newBuilder::must);
+        changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot);
+        changed |= rewriteClauses(queryRewriteContext, filterClauses, newBuilder::filter);
+        changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should);
+
+        if (changed) {
+            newBuilder.adjustPureNegative = adjustPureNegative;
+            newBuilder.disableCoord = disableCoord;
+            newBuilder.minimumShouldMatch = minimumShouldMatch;
+            newBuilder.boost(boost());
+            newBuilder.queryName(queryName());
+            return newBuilder;
+        }
+        return this;
+    }
+
+    private static boolean rewriteClauses(QueryRewriteContext queryRewriteContext, List<QueryBuilder<?>> builders, Consumer<QueryBuilder<?>> consumer) throws IOException {
+        boolean changed = false;
+        for (QueryBuilder builder : builders) {
+            QueryBuilder result = builder.rewrite(queryRewriteContext);
+            if (result != builder) {
+                changed = true;
+            }
+            consumer.accept(result);
+        }
+        return changed;
+    }
 }

core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java

@@ -158,4 +158,16 @@ public class BoostingQueryBuilder extends AbstractQueryBuilder<BoostingQueryBuil
         out.writeQuery(negativeQuery);
         out.writeFloat(negativeBoost);
     }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        QueryBuilder positiveQuery = this.positiveQuery.rewrite(queryRewriteContext);
+        QueryBuilder negativeQuery = this.negativeQuery.rewrite(queryRewriteContext);
+        if (positiveQuery != this.positiveQuery || negativeQuery != this.negativeQuery) {
+            BoostingQueryBuilder newQueryBuilder = new BoostingQueryBuilder(positiveQuery, negativeQuery);
+            newQueryBuilder.negativeBoost = negativeBoost;
+            return newQueryBuilder;
+        }
+        return this;
+    }
 }

core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java

@@ -104,4 +104,13 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
     protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeQuery(filterBuilder);
     }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        QueryBuilder rewrite = filterBuilder.rewrite(queryRewriteContext);
+        if (rewrite != filterBuilder) {
+            return new ConstantScoreQueryBuilder(rewrite);
+        }
+        return this;
+    }
 }

core/src/main/java/org/elasticsearch/index/query/EmptyQueryBuilder.java

@@ -19,12 +19,11 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 
@@ -32,80 +31,50 @@ import java.io.IOException;
  * A {@link QueryBuilder} that is a stand in replacement for an empty query clause in the DSL.
  * The current DSL allows parsing inner queries / filters like "{ }", in order to have a
  * valid non-null representation of these clauses that actually do nothing we can use this class.
- *
- * This builder has no corresponding parser and it is not registered under the query name. It is
- * intended to be used internally as a stand-in for nested queries that are left empty and should
- * be ignored upstream.
  */
-public class EmptyQueryBuilder extends ToXContentToBytes implements QueryBuilder<EmptyQueryBuilder> {
+public final class EmptyQueryBuilder extends AbstractQueryBuilder<EmptyQueryBuilder> {
 
     public static final String NAME = "empty_query";
 
     /** the one and only empty query builder */
     public static final EmptyQueryBuilder PROTOTYPE = new EmptyQueryBuilder();
 
-    // prevent instances other than prototype
-    private EmptyQueryBuilder() {
-        super(XContentType.JSON);
-    }
-
     @Override
     public String getWriteableName() {
         return NAME;
     }
 
+    @Override
+    protected Query doToQuery(QueryShardContext context) throws IOException {
+        return null;
+    }
+
     @Override
     public String getName() {
         return getWriteableName();
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.endObject();
-        return builder;
     }
 
     @Override
-    public Query toQuery(QueryShardContext context) throws IOException {
+    protected void doWriteTo(StreamOutput out) throws IOException {
-        // empty
+    }
-        return null;
+
+
+    @Override
+    protected EmptyQueryBuilder doReadFrom(StreamInput in) throws IOException {
+        return new EmptyQueryBuilder();
     }
 
     @Override
-    public Query toFilter(QueryShardContext context) throws IOException {
+    protected int doHashCode() {
-        // empty
+        return 31;
-        return null;
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    protected boolean doEquals(EmptyQueryBuilder other) {
-    }
+        return true;
-
-    @Override
-    public EmptyQueryBuilder readFrom(StreamInput in) throws IOException {
-        return EmptyQueryBuilder.PROTOTYPE;
-    }
-
-    @Override
-    public EmptyQueryBuilder queryName(String queryName) {
-        //no-op
-        return this;
-    }
-
-    @Override
-    public String queryName() {
-        return null;
-    }
-
-    @Override
-    public float boost() {
-        return -1;
-    }
-
-    @Override
-    public EmptyQueryBuilder boost(float boost) {
-        //no-op
-        return this;
     }
 }

core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java

@@ -43,7 +43,6 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper;
-import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -62,7 +61,7 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
 
     private final String fieldName;
 
-    private ShapeBuilder shape;
+    private final ShapeBuilder shape;
 
     private SpatialStrategy strategy;
 
@@ -236,13 +235,12 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
     }
 
     @Override
-    protected Query doToQuery(QueryShardContext context) throws IOException {
+    protected Query doToQuery(QueryShardContext context) {
-        ShapeBuilder shapeToQuery = shape;
+        if (shape == null) {
-        if (shapeToQuery == null) {
+            throw new UnsupportedOperationException("query must be rewritten first");
-            GetRequest getRequest = new GetRequest(indexedShapeIndex, indexedShapeType, indexedShapeId);
-            shapeToQuery = fetch(context.getClient(), getRequest, indexedShapePath);
         }
-        MappedFieldType fieldType = context.fieldMapper(fieldName);
+        final ShapeBuilder shapeToQuery = shape;
+        final MappedFieldType fieldType = context.fieldMapper(fieldName);
         if (fieldType == null) {
             throw new QueryShardException(context, "Failed to find geo_shape field [" + fieldName + "]");
         }
@@ -252,7 +250,7 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
             throw new QueryShardException(context, "Field [" + fieldName + "] is not a geo_shape");
         }
 
-        GeoShapeFieldMapper.GeoShapeFieldType shapeFieldType = (GeoShapeFieldMapper.GeoShapeFieldType) fieldType;
+        final GeoShapeFieldMapper.GeoShapeFieldType shapeFieldType = (GeoShapeFieldMapper.GeoShapeFieldType) fieldType;
 
         PrefixTreeStrategy strategy = shapeFieldType.defaultStrategy();
         if (this.strategy != null) {
@@ -449,4 +447,14 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
     public String getWriteableName() {
         return NAME;
     }
+
+    @Override
+    protected QueryBuilder<GeoShapeQueryBuilder> doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+        if (this.shape == null) {
+            GetRequest getRequest = new GetRequest(indexedShapeIndex, indexedShapeType, indexedShapeId);
+            ShapeBuilder shape = fetch(queryShardContext.getClient(), getRequest, indexedShapePath);
+            return new GeoShapeQueryBuilder(this.fieldName, shape).relation(relation).strategy(strategy);
+        }
+        return this;
+    }
 }

core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java

@@ -26,7 +26,6 @@ import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.JoinUtil;
 import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.Queries;
@@ -397,4 +396,18 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
             out.writeBoolean(false);
         }
     }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        QueryBuilder rewrite = query.rewrite(queryRewriteContext);
+        if (rewrite != query) {
+            HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder(type, rewrite);
+            hasChildQueryBuilder.minChildren = minChildren;
+            hasChildQueryBuilder.maxChildren = maxChildren;
+            hasChildQueryBuilder.scoreMode = scoreMode;
+            hasChildQueryBuilder.queryInnerHits = queryInnerHits;
+            return hasChildQueryBuilder;
+        }
+        return this;
+    }
 }

core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java

@@ -22,7 +22,6 @@ import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.Queries;
@@ -256,4 +255,16 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
     protected int doHashCode() {
         return Objects.hash(query, type, score, innerHit);
     }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+        QueryBuilder rewrite = query.rewrite(queryShardContext);
+        if (rewrite != query) {
+            HasParentQueryBuilder hasParentQueryBuilder = new HasParentQueryBuilder(type, rewrite);
+            hasParentQueryBuilder.score = score;
+            hasParentQueryBuilder.innerHit = innerHit;
+            return hasParentQueryBuilder;
+        }
+        return this;
+    }
 }

core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java

@@ -140,4 +140,14 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder<IndicesQueryBuilde
                 Arrays.equals(indices, other.indices) &&  // otherwise we are comparing pointers
                 Objects.equals(noMatchQuery, other.noMatchQuery);
     }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+        QueryBuilder<?> newInnnerQuery = innerQuery.rewrite(queryShardContext);
+        QueryBuilder<?> newNoMatchQuery = noMatchQuery.rewrite(queryShardContext);
+        if (newInnnerQuery != innerQuery || newNoMatchQuery != noMatchQuery) {
+            return new IndicesQueryBuilder(innerQuery, indices).noMatchQuery(noMatchQuery);
+        }
+        return this;
+    }
 }

core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java

@@ -1050,4 +1050,10 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
                 Objects.equals(include, other.include) &&
                 Objects.equals(failOnUnsupportedField, other.failOnUnsupportedField);
     }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        // TODO this needs heavy cleanups before we can rewrite it
+        return this;
+    }
 }

core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java

@@ -225,4 +225,12 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
         return new ToParentBlockJoinQuery(Queries.filtered(innerQuery, childFilter), parentFilter, scoreMode);
     }
 
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        QueryBuilder rewrite = query.rewrite(queryRewriteContext);
+        if (rewrite != query) {
+            return new NestedQueryBuilder(path, rewrite).scoreMode(scoreMode);
+        }
+        return this;
+    }
 }

core/src/main/java/org/elasticsearch/index/query/QueryBuilder.java

@@ -72,4 +72,28 @@ public interface QueryBuilder<QB extends QueryBuilder<QB>> extends NamedWriteabl
      * Returns the name that identifies uniquely the query
      */
     String getName();
+
+    /**
+     * Rewrites this query builder into its primitive form. By default this method return the builder itself. If the builder
+     * did not change the identity reference must be returned otherwise the builder will be rewritten infinitely.
+     */
+    default QueryBuilder<?> rewrite(QueryRewriteContext queryShardContext) throws IOException {
+        return this;
+    }
+
+    /**
+     * Rewrites the given query into its primitive form. Queries that for instance fetch resources from remote hosts or
+     * can simplify / optimize itself should do their heavy lifting during {@link #rewrite(QueryRewriteContext)}. This method
+     * rewrites the query until it doesn't change anymore.
+     * @throws IOException if an {@link IOException} occurs
+     */
+    static QueryBuilder<?> rewriteQuery(QueryBuilder<?> original, QueryRewriteContext context) throws IOException {
+        QueryBuilder builder = original;
+        for (QueryBuilder rewrittenBuilder = builder.rewrite(context); rewrittenBuilder != builder;
+             rewrittenBuilder = builder.rewrite(context)) {
+            builder = rewrittenBuilder;
+        }
+        return builder;
+    }
+
 }

core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java

@@ -106,7 +106,7 @@ public class QueryParseContext {
         token = parser.nextToken();
         if (token == XContentParser.Token.END_OBJECT) {
             // empty query
-            return EmptyQueryBuilder.PROTOTYPE;
+            return new EmptyQueryBuilder();
         }
         if (token != XContentParser.Token.FIELD_NAME) {
             throw new ParsingException(parser.getTokenLocation(), "[_na] query malformed, no field after start_object");

core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.script.ScriptService;
+
+/**
+ * Context object used to rewrite {@link QueryBuilder} instances into simplified version.
+ */
+public class QueryRewriteContext {
+    protected final ScriptService scriptService;
+    protected final IndexSettings indexSettings;
+    protected final IndicesQueriesRegistry indicesQueriesRegistry;
+    protected final QueryParseContext parseContext;
+
+    public QueryRewriteContext(IndexSettings indexSettings, ScriptService scriptService, IndicesQueriesRegistry indicesQueriesRegistry) {
+        this.scriptService = scriptService;
+        this.indexSettings = indexSettings;
+        this.indicesQueriesRegistry = indicesQueriesRegistry;
+        this.parseContext = new QueryParseContext(indicesQueriesRegistry);
+    }
+
+    /**
+     * Returns a clients to fetch resources from local or remove nodes.
+     */
+    public final Client getClient() {
+        return scriptService.getClient();
+    }
+
+    /**
+     * Returns the index settings for this context. This might return null if the
+     * context has not index scope.
+     */
+    public final IndexSettings getIndexSettings() {
+        return indexSettings;
+    }
+
+    /**
+     * Returns a script service to fetch scripts.
+     */
+    public final ScriptService getScriptService() {
+        return scriptService;
+    }
+
+    /**
+     * Returns a new {@link QueryParseContext} to parse template or wrapped queries.
+     */
+    public QueryParseContext newParseContext() {
+        QueryParseContext queryParseContext = new QueryParseContext(indicesQueriesRegistry);
+        queryParseContext.parseFieldMatcher(parseContext.parseFieldMatcher());
+        return queryParseContext;
+    }
+}

core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java

@@ -26,7 +26,6 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.search.similarities.Similarity;
 import org.elasticsearch.Version;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
@@ -51,10 +50,7 @@ import org.elasticsearch.index.query.support.InnerHitsQueryParserHelper;
 import org.elasticsearch.index.query.support.NestedScope;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
 import org.elasticsearch.search.fetch.innerhits.InnerHitsSubSearchContext;
 import org.elasticsearch.search.internal.SearchContext;
@@ -63,7 +59,6 @@ import org.elasticsearch.search.lookup.SearchLookup;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -72,15 +67,13 @@ import static java.util.Collections.unmodifiableMap;
 /**
  * Context object used to create lucene queries on the shard level.
  */
-public class QueryShardContext {
+public class QueryShardContext extends QueryRewriteContext {
 
     private final MapperService mapperService;
-    private final ScriptService scriptService;
     private final SimilarityService similarityService;
     private final BitsetFilterCache bitsetFilterCache;
     private final IndexFieldDataService indexFieldDataService;
     private final IndexSettings indexSettings;
-    private final Client client;
     private String[] types = Strings.EMPTY_ARRAY;
 
     public void setTypes(String... types) {
@@ -93,35 +86,31 @@ public class QueryShardContext {
 
     private final Map<String, Query> namedQueries = new HashMap<>();
     private final MapperQueryParser queryParser = new MapperQueryParser(this);
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
     private boolean allowUnmappedFields;
     private boolean mapUnmappedFieldAsString;
     private NestedScope nestedScope;
-    private QueryParseContext parseContext;
     boolean isFilter; // pkg private for testing
 
-    public QueryShardContext(IndexSettings indexSettings, Client client, BitsetFilterCache bitsetFilterCache, IndexFieldDataService indexFieldDataService, MapperService mapperService, SimilarityService similarityService, ScriptService scriptService,
+    public QueryShardContext(IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, IndexFieldDataService indexFieldDataService, MapperService mapperService, SimilarityService similarityService, ScriptService scriptService,
                              final IndicesQueriesRegistry indicesQueriesRegistry) {
+        super(indexSettings, scriptService, indicesQueriesRegistry);
         this.indexSettings = indexSettings;
-        this.scriptService = scriptService;
-        this.client = client;
         this.similarityService = similarityService;
         this.mapperService = mapperService;
         this.bitsetFilterCache = bitsetFilterCache;
         this.indexFieldDataService = indexFieldDataService;
         this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields();
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
+
-        this.parseContext = new QueryParseContext(indicesQueriesRegistry);
     }
 
     public QueryShardContext(QueryShardContext source) {
-        this(source.indexSettings, source.client, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.indicesQueriesRegistry);
+        this(source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.indicesQueriesRegistry);
         this.types = source.getTypes();
     }
 
 
     public QueryShardContext clone() {
-        return new QueryShardContext(indexSettings, client, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry);
+        return new QueryShardContext(indexSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry);
     }
 
     public void parseFieldMatcher(ParseFieldMatcher parseFieldMatcher) {
@@ -146,10 +135,6 @@ public class QueryShardContext {
         this.parseContext.reset(jp);
     }
 
-    public Index index() {
-        return this.mapperService.getIndexSettings().getIndex();
-    }
-
     public InnerHitsSubSearchContext getInnerHitsContext(XContentParser parser) throws IOException {
         return InnerHitsQueryParserHelper.parse(parser);
     }
@@ -158,10 +143,6 @@ public class QueryShardContext {
         return mapperService.analysisService();
     }
 
-    public ScriptService getScriptService() {
-        return scriptService;
-    }
-
     public MapperService getMapperService() {
         return mapperService;
     }
@@ -210,10 +191,6 @@ public class QueryShardContext {
         return unmodifiableMap(new HashMap<>(namedQueries));
     }
 
-    public void combineNamedQueries(QueryShardContext context) {
-        namedQueries.putAll(context.namedQueries);
-    }
-
     /**
      * Return whether we are currently parsing a filter or a query.
      */
@@ -340,18 +317,6 @@ public class QueryShardContext {
         return false;
     }
 
-    /*
-    * Executes the given template, and returns the response.
-    */
-    public BytesReference executeQueryTemplate(Template template) {
-        ExecutableScript executable = getScriptService().executable(template, ScriptContext.Standard.SEARCH, Collections.emptyMap());
-        return (BytesReference) executable.run();
-    }
-
-    public Client getClient() {
-        return client;
-    }
-
     public ParsedQuery parse(BytesReference source) {
         XContentParser parser = null;
         try {
@@ -384,7 +349,7 @@ public class QueryShardContext {
         reset(parser);
         try {
             parseFieldMatcher(indexSettings.getParseFieldMatcher());
-            Query filter = parseContext().parseInnerQueryBuilder().toFilter(this);
+            Query filter = QueryBuilder.rewriteQuery(parseContext().parseInnerQueryBuilder(), this).toFilter(this);
             if (filter == null) {
                 return null;
             }
@@ -425,12 +390,16 @@ public class QueryShardContext {
         }
     }
 
-    private static Query toQuery(QueryBuilder<?> queryBuilder, QueryShardContext context) throws IOException {
+    private static Query toQuery(final QueryBuilder<?> queryBuilder, final QueryShardContext context) throws IOException {
-        Query query = queryBuilder.toQuery(context);
+        final Query query = QueryBuilder.rewriteQuery(queryBuilder, context).toQuery(context);
         if (query == null) {
-            query = Queries.newMatchNoDocsQuery();
+            return Queries.newMatchNoDocsQuery();
         }
         return query;
     }
 
+    public final Index index() {
+        return indexSettings.getIndex();
+    }
+
 }

core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java

@@ -25,11 +25,13 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.Template;
-import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
 
@@ -100,14 +102,7 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder<TemplateQueryBuil
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
-        BytesReference querySource = context.executeQueryTemplate(template);
+        throw new UnsupportedOperationException("this query must be rewritten first");
-        try (XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource)) {
-            final QueryShardContext contextCopy = new QueryShardContext(context);
-            contextCopy.reset(qSourceParser);
-            QueryBuilder result = contextCopy.parseContext().parseInnerQueryBuilder();
-            context.combineNamedQueries(contextCopy);
-            return result.toQuery(context);
-        }
     }
 
     @Override
@@ -130,4 +125,22 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder<TemplateQueryBuil
     protected boolean doEquals(TemplateQueryBuilder other) {
         return Objects.equals(template, other.template);
     }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        ExecutableScript executable = queryRewriteContext.getScriptService().executable(template,
+            ScriptContext.Standard.SEARCH, Collections.emptyMap());
+        BytesReference querySource = (BytesReference) executable.run();
+        final QueryParseContext queryParseContext = queryRewriteContext.newParseContext();
+        try (XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource)) {
+            queryParseContext.reset(qSourceParser);
+            final QueryBuilder<?> queryBuilder = queryParseContext.parseInnerQueryBuilder();
+            if (boost() != DEFAULT_BOOST || queryName() != null) {
+                final BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+                boolQueryBuilder.must(queryBuilder);
+                return boolQueryBuilder;
+            }
+            return queryBuilder;
+        }
+    }
 }

core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java

@@ -226,22 +226,13 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
-        List<Object> terms;
+        if (termsLookup != null) {
-        TermsLookup termsLookup = null;
+            throw new UnsupportedOperationException("query must be rewritten first");
-        if (this.termsLookup != null) {
-            termsLookup = new TermsLookup(this.termsLookup);
-            if (termsLookup.index() == null) {
-                termsLookup.index(context.index().getName());
-            }
-            Client client = context.getClient();
-            terms = fetch(termsLookup, client);
-        } else {
-            terms = values;
         }
-        if (terms == null || terms.isEmpty()) {
+        if (values == null || values.isEmpty()) {
             return Queries.newMatchNoDocsQuery();
         }
-        return handleTermsQuery(terms, fieldName, context);
+        return handleTermsQuery(values, fieldName, context);
     }
 
     private List<Object> fetch(TermsLookup termsLookup, Client client) {
@@ -323,4 +314,22 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
                 Objects.equals(values, other.values) &&
                 Objects.equals(termsLookup, other.termsLookup);
     }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        if (this.termsLookup != null) {
+            TermsLookup termsLookup = new TermsLookup(this.termsLookup);
+            if (termsLookup.index() == null) { // TODO this should go away?
+                if (queryRewriteContext.getIndexSettings() != null) {
+                    termsLookup.index(queryRewriteContext.getIndexSettings().getIndex().getName());
+                } else {
+                    return this; // can't rewrite until we have index scope on the shard
+                }
+            }
+            List<Object> values = fetch(termsLookup, queryRewriteContext.getClient());
+            return new TermsQueryBuilder(this.fieldName, values);
+        }
+        return this;
+    }
+
 }

core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java

@@ -105,14 +105,7 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder<WrapperQueryBuilde
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
-        try (XContentParser qSourceParser = XContentFactory.xContent(source).createParser(source)) {
+        throw new UnsupportedOperationException("this query must be rewritten first");
-            final QueryShardContext contextCopy = new QueryShardContext(context);
-            contextCopy.reset(qSourceParser);
-            contextCopy.parseFieldMatcher(context.parseFieldMatcher());
-            QueryBuilder<?> result = contextCopy.parseContext().parseInnerQueryBuilder();
-            context.combineNamedQueries(contextCopy);
-            return result.toQuery(context);
-        }
     }
 
     @Override
@@ -134,4 +127,22 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder<WrapperQueryBuilde
     protected boolean doEquals(WrapperQueryBuilder other) {
         return Arrays.equals(source, other.source);   // otherwise we compare pointers
     }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext context) throws IOException {
+        try (XContentParser qSourceParser = XContentFactory.xContent(source).createParser(source)) {
+            QueryParseContext parseContext = context.newParseContext();
+            parseContext.reset(qSourceParser);
+
+            final QueryBuilder<?> queryBuilder = parseContext.parseInnerQueryBuilder();
+            if (boost() != DEFAULT_BOOST || queryName() != null) {
+                final BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+                boolQueryBuilder.must(queryBuilder);
+                return boolQueryBuilder;
+            }
+            return queryBuilder;
+        }
+    }
+
+
 }

core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java

@@ -34,6 +34,7 @@ import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.EmptyQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.functionscore.random.RandomScoreFunctionBuilder;
 
@@ -393,5 +394,33 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
         public FilterFunctionBuilder readFrom(StreamInput in) throws IOException {
             return new FilterFunctionBuilder(in.readQuery(), in.readScoreFunction());
         }
+
+        public FilterFunctionBuilder rewrite(QueryRewriteContext context) throws IOException {
+            QueryBuilder<?> rewrite = filter.rewrite(context);
+            if (rewrite != filter) {
+                return new FilterFunctionBuilder(rewrite, scoreFunction);
+            }
+            return this;
+        }
+    }
+
+    @Override
+    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        QueryBuilder<?> queryBuilder = this.query.rewrite(queryRewriteContext);
+        FilterFunctionBuilder[] rewrittenBuilders = new FilterFunctionBuilder[this.filterFunctionBuilders.length];
+        boolean rewritten = false;
+        for (int i = 0; i < rewrittenBuilders.length; i++) {
+            FilterFunctionBuilder rewrite = filterFunctionBuilders[i].rewrite(queryRewriteContext);
+            rewritten |= rewrite != filterFunctionBuilders[i];
+            rewrittenBuilders[i] = rewrite;
+        }
+        if (queryBuilder != query || rewritten) {
+            FunctionScoreQueryBuilder newQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder, rewrittenBuilders);
+            newQueryBuilder.scoreMode = scoreMode;
+            newQueryBuilder.minScore = minScore;
+            newQueryBuilder.maxBoost = maxBoost;
+            return newQueryBuilder;
+        }
+        return this;
     }
 }

core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java

@@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
@@ -91,7 +92,8 @@ public class NestedInnerQueryParseSupport {
         if (path != null) {
             setPathLevel();
             try {
-                innerFilter = parseContext.parseInnerQueryBuilder().toFilter(this.shardContext);
+                innerFilter = QueryBuilder.rewriteQuery(parseContext.parseInnerQueryBuilder(),
+                    this.shardContext).toFilter(this.shardContext);
             } finally {
                 resetPathLevel();
             }
@@ -147,7 +149,8 @@ public class NestedInnerQueryParseSupport {
             try {
                 XContentParser innerParser = XContentHelper.createParser(source);
                 parseContext.parser(innerParser);
-                innerFilter = parseContext.parseInnerQueryBuilder().toFilter(this.shardContext);
+                innerFilter = QueryBuilder.rewriteQuery(parseContext.parseInnerQueryBuilder(),
+                    this.shardContext).toFilter(this.shardContext);
                 filterParsed = true;
                 return innerFilter;
             } finally {

core/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -248,7 +248,7 @@ public class IndexShard extends AbstractIndexShardComponent {
         this.engineConfig = newEngineConfig(translogConfig, cachingPolicy);
         this.suspendableRefContainer = new SuspendableRefContainer();
         this.searcherWrapper = indexSearcherWrapper;
-        QueryShardContext queryShardContext = new QueryShardContext(idxSettings, provider.getClient(), indexCache.bitsetFilterCache(), indexFieldDataService, mapperService, similarityService, provider.getScriptService(), provider.getIndicesQueriesRegistry());
+        QueryShardContext queryShardContext = new QueryShardContext(idxSettings, indexCache.bitsetFilterCache(), indexFieldDataService, mapperService, similarityService, provider.getScriptService(), provider.getIndicesQueriesRegistry());
         this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, queryShardContext);
     }
 

core/src/main/java/org/elasticsearch/script/ScriptService.java

@@ -489,6 +489,10 @@ public class ScriptService extends AbstractComponent implements Closeable {
         return scriptMetrics.stats();
     }
 
+    public Client getClient() {
+        return client;
+    }
+
     /**
      * A small listener for the script cache that calls each
      * {@code ScriptEngineService}'s {@code scriptRemoved} method when the

core/src/main/java/org/elasticsearch/search/SearchService.java

@@ -532,8 +532,10 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
 
         DefaultSearchContext context = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget, engineSearcher, indexService, indexShard, scriptService, pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher, defaultSearchTimeout);
         SearchContext.setCurrent(context);
-
         try {
+            if (request.source() != null) {
+                request.source().rewrite(context.getQueryShardContext());
+            }
             if (request.scroll() != null) {
                 context.scrollContext(new ScrollContext());
                 context.scrollContext().scroll = request.scroll();

core/src/main/java/org/elasticsearch/search/action/SearchServiceTransportAction.java

@@ -81,7 +81,6 @@ public class SearchServiceTransportAction extends AbstractComponent {
         super(settings);
         this.transportService = transportService;
         this.searchService = searchService;
-
         transportService.registerRequestHandler(FREE_CONTEXT_SCROLL_ACTION_NAME, ScrollFreeContextRequest::new, ThreadPool.Names.SAME, new FreeContextTransportHandler<>());
         transportService.registerRequestHandler(FREE_CONTEXT_ACTION_NAME, SearchFreeContextRequest::new, ThreadPool.Names.SAME, new FreeContextTransportHandler<SearchFreeContextRequest>());
         transportService.registerRequestHandler(CLEAR_SCROLL_CONTEXTS_ACTION_NAME, ClearScrollContextsRequest::new, ThreadPool.Names.SAME, new ClearScrollContextsTransportHandler());

core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java

@@ -40,6 +40,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
@@ -1433,4 +1434,17 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 && Objects.equals(version, other.version)
                 && Objects.equals(profile, other.profile);
     }
+
+    /**
+     * Rewrites the internal query builders in-place
+     */
+    public void rewrite(QueryRewriteContext rewriteContext) throws IOException {
+        if (queryBuilder != null) {
+            queryBuilder = QueryBuilder.rewriteQuery(queryBuilder, rewriteContext);
+        }
+        if (postQueryBuilder != null) {
+            postQueryBuilder = QueryBuilder.rewriteQuery(postQueryBuilder, rewriteContext);
+        }
+    }
+
 }

core/src/main/java/org/elasticsearch/search/highlight/HighlightBuilder.java

@@ -355,7 +355,7 @@ public class HighlightBuilder extends AbstractHighlighterBuilder<HighlightBuilde
             targetOptionsBuilder.options(highlighterBuilder.options);
         }
         if (highlighterBuilder.highlightQuery != null) {
-            targetOptionsBuilder.highlightQuery(highlighterBuilder.highlightQuery.toQuery(context));
+            targetOptionsBuilder.highlightQuery(QueryBuilder.rewriteQuery(highlighterBuilder.highlightQuery, context).toQuery(context));
         }
     }
 

core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java

@@ -149,7 +149,7 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
     public QueryRescoreContext build(QueryShardContext context) throws IOException {
         org.elasticsearch.search.rescore.QueryRescorer rescorer = new org.elasticsearch.search.rescore.QueryRescorer();
         QueryRescoreContext queryRescoreContext = new QueryRescoreContext(rescorer);
-        queryRescoreContext.setQuery(this.queryBuilder.toQuery(context));
+        queryRescoreContext.setQuery(QueryBuilder.rewriteQuery(this.queryBuilder, context).toQuery(context));
         queryRescoreContext.setQueryWeight(this.queryWeight);
         queryRescoreContext.setRescoreQueryWeight(this.rescoreQueryWeight);
         queryRescoreContext.setScoreMode(this.scoreMode);
@@ -239,4 +239,4 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
             this.scoreMode = scoreMode;
         }
     }
-}
+}

core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java

@@ -23,6 +23,7 @@ import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
 
+import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -286,7 +287,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             }
         });
         indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
-        queryShardContext = new QueryShardContext(idxSettings, proxy, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry);
+        queryShardContext = new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry);
         //create some random type with some default field, those types will stick around for all of the subclasses
         currentTypes = new String[randomIntBetween(0, 5)];
         for (int i = 0; i < currentTypes.length; i++) {
@@ -516,7 +517,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             QB firstQuery = createTestQueryBuilder();
             QB controlQuery = copyQuery(firstQuery);
             setSearchContext(randomTypes); // only set search context for toQuery to be more realistic
-            Query firstLuceneQuery = firstQuery.toQuery(context);
+            Query firstLuceneQuery = rewriteQuery(firstQuery, context).toQuery(context);
             assertLuceneQuery(firstQuery, firstLuceneQuery, context);
             SearchContext.removeCurrent(); // remove after assertLuceneQuery since the assertLuceneQuery impl might access the context as well
             assertTrue(
@@ -534,24 +535,31 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                         + randomAsciiOfLengthBetween(1, 10));
             }
             setSearchContext(randomTypes);
-            Query secondLuceneQuery = secondQuery.toQuery(context);
+            Query secondLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context);
             assertLuceneQuery(secondQuery, secondLuceneQuery, context);
             SearchContext.removeCurrent();
 
-            assertThat("two equivalent query builders lead to different lucene queries", secondLuceneQuery, equalTo(firstLuceneQuery));
+            assertEquals("two equivalent query builders lead to different lucene queries", rewrite(secondLuceneQuery), rewrite(firstLuceneQuery));
 
             // if the initial lucene query is null, changing its boost won't have any effect, we shouldn't test that
             if (firstLuceneQuery != null && supportsBoostAndQueryName()) {
                 secondQuery.boost(firstQuery.boost() + 1f + randomFloat());
                 setSearchContext(randomTypes);
-                Query thirdLuceneQuery = secondQuery.toQuery(context);
+                Query thirdLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context);
                 SearchContext.removeCurrent();
-                assertThat("modifying the boost doesn't affect the corresponding lucene query", firstLuceneQuery,
+                assertNotEquals("modifying the boost doesn't affect the corresponding lucene query", rewrite(firstLuceneQuery),
-                        not(equalTo(thirdLuceneQuery)));
+                        rewrite(thirdLuceneQuery));
             }
         }
     }
 
+    private QueryBuilder<?> rewriteQuery(QB queryBuilder, QueryRewriteContext rewriteContext) throws IOException {
+        QueryBuilder<?> rewritten = QueryBuilder.rewriteQuery(queryBuilder, rewriteContext);
+        // extra safety to fail fast - serialize the rewritten version to ensure it's serializable.
+        assertSerialization(rewritten);
+        return rewritten;
+    }
+
     /**
      * Few queries allow you to set the boost and queryName on the java api, although the corresponding parser doesn't parse them as they are not supported.
      * This method allows to disable boost and queryName related tests for those queries. Those queries are easy to identify: their parsers
@@ -625,11 +633,13 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
      * Serialize the given query builder and asserts that both are equal
      */
     @SuppressWarnings("unchecked")
-    protected QB assertSerialization(QB testQuery) throws IOException {
+    protected <QB extends QueryBuilder> QB assertSerialization(QB testQuery) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             testQuery.writeTo(output);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
-                QueryBuilder<?> prototype = queryParser(testQuery.getName()).getBuilderPrototype();
+                QueryParser<?> queryParser = queryParser(testQuery.getName());
+                assertNotNull("queryparser not found for query: [" + testQuery.getName() + "]", queryParser);
+                QueryBuilder<?> prototype = queryParser.getBuilderPrototype();
                 QueryBuilder<?> deserializedQuery = prototype.readFrom(in);
                 assertEquals(deserializedQuery, testQuery);
                 assertEquals(deserializedQuery.hashCode(), testQuery.hashCode());
@@ -674,7 +684,26 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     }
 
     private QueryParser<?> queryParser(String queryId) {
-        return indicesQueriesRegistry.queryParsers().get(queryId);
+        QueryParser<?> queryParser = indicesQueriesRegistry.queryParsers().get(queryId);
+        if (queryParser == null && EmptyQueryBuilder.NAME.equals(queryId)) {
+            return new QueryParser() {
+                @Override
+                public String[] names() {
+                    return new String[] {EmptyQueryBuilder.NAME};
+                }
+
+                @Override
+                public QueryBuilder<?> fromXContent(QueryParseContext parseContext) throws IOException {
+                    return new EmptyQueryBuilder();
+                }
+
+                @Override
+                public QueryBuilder getBuilderPrototype() {
+                    return EmptyQueryBuilder.PROTOTYPE;
+                }
+            };
+        }
+        return queryParser;
     }
 
     //we use the streaming infra to create a copy of the query provided as argument
@@ -948,4 +977,21 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         }
         return "";
     }
+
+    /**
+     * This test ensures that queries that need to be rewritten have dedicated tests.
+     * These queries must override this method accordingly.
+     */
+    public void testMustRewrite() throws IOException {
+        QueryShardContext context = createShardContext();
+        context.setAllowUnmappedFields(true);
+        QB queryBuilder = createTestQueryBuilder();
+        setSearchContext(randomTypes); // only set search context for toQuery to be more realistic
+        queryBuilder.toQuery(context);
+    }
+
+    protected Query rewrite(Query query) throws IOException {
+        return query;
+    }
+
 }

core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java

@@ -343,4 +343,70 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
         assertEquals(query, "23", queryBuilder.minimumShouldMatch());
         assertEquals(query, "kimchy", ((TermQueryBuilder)queryBuilder.must().get(0)).value());
     }
+
+    public void testRewrite() throws IOException {
+        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+        boolean mustRewrite = false;
+        if (randomBoolean()) {
+            mustRewrite = true;
+            boolQueryBuilder.must(new WrapperQueryBuilder(new TermsQueryBuilder("foo", "must").toString()));
+        }
+        if (randomBoolean()) {
+            mustRewrite = true;
+            boolQueryBuilder.should(new WrapperQueryBuilder(new TermsQueryBuilder("foo", "should").toString()));
+        }
+        if (randomBoolean()) {
+            mustRewrite = true;
+            boolQueryBuilder.filter(new WrapperQueryBuilder(new TermsQueryBuilder("foo", "filter").toString()));
+        }
+        if (randomBoolean()) {
+            mustRewrite = true;
+            boolQueryBuilder.mustNot(new WrapperQueryBuilder(new TermsQueryBuilder("foo", "must_not").toString()));
+        }
+        if (mustRewrite == false && randomBoolean()) {
+            boolQueryBuilder.must(new TermsQueryBuilder("foo", "no_rewrite"));
+        }
+        QueryBuilder<?> rewritten = boolQueryBuilder.rewrite(queryShardContext());
+        if (mustRewrite == false && boolQueryBuilder.must().isEmpty()) {
+            // if it's empty we rewrite to match all
+            assertEquals(rewritten, new MatchAllQueryBuilder());
+        } else {
+            BoolQueryBuilder rewrite = (BoolQueryBuilder) rewritten;
+            if (mustRewrite) {
+                assertNotSame(rewrite, boolQueryBuilder);
+                if (boolQueryBuilder.must().isEmpty() == false) {
+                    assertEquals(new TermsQueryBuilder("foo", "must"), rewrite.must().get(0));
+                }
+                if (boolQueryBuilder.should().isEmpty() == false) {
+                    assertEquals(new TermsQueryBuilder("foo", "should"), rewrite.should().get(0));
+                }
+                if (boolQueryBuilder.mustNot().isEmpty() == false) {
+                    assertEquals(new TermsQueryBuilder("foo", "must_not"), rewrite.mustNot().get(0));
+                }
+                if (boolQueryBuilder.filter().isEmpty() == false) {
+                    assertEquals(new TermsQueryBuilder("foo", "filter"), rewrite.filter().get(0));
+                }
+            } else {
+                assertSame(rewrite, boolQueryBuilder);
+                if (boolQueryBuilder.must().isEmpty() == false) {
+                    assertSame(boolQueryBuilder.must().get(0), rewrite.must().get(0));
+                }
+            }
+        }
+    }
+
+    public void testRewriteMultipleTimes() throws IOException {
+        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
+        boolQueryBuilder.must(new WrapperQueryBuilder(new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()).toString()));
+        QueryBuilder<?> rewritten = boolQueryBuilder.rewrite(queryShardContext());
+        BoolQueryBuilder expected = new BoolQueryBuilder();
+        expected.must(new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()));
+        assertEquals(expected, rewritten);
+
+        expected = new BoolQueryBuilder();
+        expected.must(new MatchAllQueryBuilder());
+        QueryBuilder<?> rewrittenAgain = rewritten.rewrite(queryShardContext());
+        assertEquals(rewrittenAgain, expected);
+        assertEquals(QueryBuilder.rewriteQuery(boolQueryBuilder, queryShardContext()), expected);
+    }
 }

core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.queries.BoostingQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 
 import java.io.IOException;
@@ -72,27 +73,27 @@ public class BoostingQueryBuilderTests extends AbstractQueryTestCase<BoostingQue
 
     public void testFromJson() throws IOException {
         String query =
-                "{\n" + 
+                "{\n" +
-                "  \"boosting\" : {\n" + 
+                "  \"boosting\" : {\n" +
-                "    \"positive\" : {\n" + 
+                "    \"positive\" : {\n" +
-                "      \"term\" : {\n" + 
+                "      \"term\" : {\n" +
-                "        \"field1\" : {\n" + 
+                "        \"field1\" : {\n" +
-                "          \"value\" : \"value1\",\n" + 
+                "          \"value\" : \"value1\",\n" +
-                "          \"boost\" : 5.0\n" + 
+                "          \"boost\" : 5.0\n" +
-                "        }\n" + 
+                "        }\n" +
-                "      }\n" + 
+                "      }\n" +
-                "    },\n" + 
+                "    },\n" +
-                "    \"negative\" : {\n" + 
+                "    \"negative\" : {\n" +
-                "      \"term\" : {\n" + 
+                "      \"term\" : {\n" +
-                "        \"field2\" : {\n" + 
+                "        \"field2\" : {\n" +
-                "          \"value\" : \"value2\",\n" + 
+                "          \"value\" : \"value2\",\n" +
-                "          \"boost\" : 8.0\n" + 
+                "          \"boost\" : 8.0\n" +
-                "        }\n" + 
+                "        }\n" +
-                "      }\n" + 
+                "      }\n" +
-                "    },\n" + 
+                "    },\n" +
-                "    \"negative_boost\" : 23.0,\n" + 
+                "    \"negative_boost\" : 23.0,\n" +
-                "    \"boost\" : 42.0\n" + 
+                "    \"boost\" : 42.0\n" +
-                "  }\n" + 
+                "  }\n" +
                 "}";
 
         BoostingQueryBuilder queryBuilder = (BoostingQueryBuilder) parseQuery(query);
@@ -103,4 +104,17 @@ public class BoostingQueryBuilderTests extends AbstractQueryTestCase<BoostingQue
         assertEquals(query, 8, queryBuilder.negativeQuery().boost(), 0.00001);
         assertEquals(query, 5, queryBuilder.positiveQuery().boost(), 0.00001);
     }
+
+    public void testRewrite() throws IOException {
+        QueryBuilder positive = randomBoolean() ? new MatchAllQueryBuilder() : new WrapperQueryBuilder(new TermQueryBuilder("pos", "bar").toString());
+        QueryBuilder negative = randomBoolean() ? new MatchAllQueryBuilder() : new WrapperQueryBuilder(new TermQueryBuilder("neg", "bar").toString());
+        BoostingQueryBuilder qb = new BoostingQueryBuilder(positive, negative);
+        QueryBuilder<?> rewrite = qb.rewrite(queryShardContext());
+        if (positive instanceof MatchAllQueryBuilder && negative instanceof MatchAllQueryBuilder) {
+            assertSame(rewrite, qb);
+        } else {
+            assertNotSame(rewrite, qb);
+            assertEquals(new BoostingQueryBuilder(positive.rewrite(queryShardContext()), negative.rewrite(queryShardContext())), rewrite);
+        }
+    }
 }

core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java

@@ -468,4 +468,10 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
         assertEquals(json, 1.0, parsed.boost(), 0.0001);
         assertEquals(json, GeoExecType.MEMORY, parsed.type());
     }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        super.testMustRewrite();
+    }
 }

core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java

@@ -411,4 +411,10 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
         assertEquals(json, 40.0, parsed.point().getLat(), 0.0001);
         assertEquals(json, 12000.0, parsed.distance(), 0.0001);
     }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        super.testMustRewrite();
+    }
 }

core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java

@@ -316,4 +316,10 @@ public class GeoDistanceRangeQueryTests extends AbstractQueryTestCase<GeoDistanc
         checkGeneratedJson(json, parsed);
         assertEquals(json, -70.0, parsed.point().lon(), 0.0001);
     }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        super.testMustRewrite();
+    }
 }

core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java

@@ -343,4 +343,10 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
         checkGeneratedJson(json, parsed);
         assertEquals(json, 4, parsed.points().size());
     }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        super.testMustRewrite();
+    }
 }

core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java

@@ -240,4 +240,24 @@ public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase<GeoShapeQue
         checkGeneratedJson(json, parsed);
         assertEquals(json, 42.0, parsed.boost(), 0.0001);
     }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        GeoShapeQueryBuilder sqb;
+        do {
+            sqb = doCreateTestQueryBuilder();
+            // do this until we get one without a shape
+        } while (sqb.shape() != null);
+        try {
+            sqb.toQuery(queryShardContext());
+            fail();
+        } catch (UnsupportedOperationException e) {
+            assertEquals("query must be rewritten first", e.getMessage());
+        }
+        QueryBuilder<?> rewrite = sqb.rewrite(queryShardContext());
+        GeoShapeQueryBuilder geoShapeQueryBuilder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeToReturn);
+        geoShapeQueryBuilder.strategy(sqb.strategy());
+        geoShapeQueryBuilder.relation(sqb.relation());
+        assertEquals(geoShapeQueryBuilder, rewrite);
+    }
 }

core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java

@@ -145,4 +145,10 @@ public class GeohashCellQueryBuilderTests extends AbstractQueryTestCase<Builder>
         checkGeneratedJson(json, parsed);
         assertEquals(json, 3, parsed.precision().intValue());
     }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        super.testMustRewrite();
+    }
 }

core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java

@@ -21,15 +21,12 @@ package org.elasticsearch.index.query;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Collections;
-
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
@@ -50,7 +47,7 @@ public class QueryShardContextTests extends ESTestCase {
         MapperService mapperService = mock(MapperService.class);
         when(mapperService.getIndexSettings()).thenReturn(indexSettings);
         QueryShardContext context = new QueryShardContext(
-            indexSettings, null, null, null, mapperService, null, null, null
+            indexSettings, null, null, mapperService, null, null, null
         );
 
         context.setAllowUnmappedFields(false);

core/src/test/java/org/elasticsearch/index/query/TemplateQueryBuilderTests.java

@@ -19,9 +19,12 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.script.Script.ScriptParseException;
 import org.elasticsearch.script.ScriptService.ScriptType;
@@ -29,6 +32,7 @@ import org.elasticsearch.script.Template;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -56,7 +60,7 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
 
     @Override
     protected void doAssertLuceneQuery(TemplateQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        assertEquals(templateBase.toQuery(context), query);
+        assertEquals(rewrite(QueryBuilder.rewriteQuery(templateBase, context).toQuery(context)), rewrite(query));
     }
 
     public void testIllegalArgument() {
@@ -118,4 +122,53 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
                 XContentType.JSON, params));
         assertParsedQuery(query, expectedBuilder);
     }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        String query = "{ \"match_all\" : {}}";
+        QueryBuilder<?> builder = new TemplateQueryBuilder(new Template(query, ScriptType.INLINE, "mockscript",
+        XContentType.JSON, Collections.emptyMap()));
+        try {
+            builder.toQuery(queryShardContext());
+            fail();
+        } catch (UnsupportedOperationException ex) {
+            assertEquals("this query must be rewritten first", ex.getMessage());
+        }
+        assertEquals(new MatchAllQueryBuilder(), builder.rewrite(queryShardContext()));
+    }
+
+    public void testRewriteWithInnerName() throws IOException {
+        final String query = "{ \"match_all\" : {\"_name\" : \"foobar\"}}";
+        QueryBuilder<?> builder = new TemplateQueryBuilder(new Template(query, ScriptType.INLINE, "mockscript",
+            XContentType.JSON, Collections.emptyMap()));
+        assertEquals(new MatchAllQueryBuilder().queryName("foobar"), builder.rewrite(queryShardContext()));
+
+        builder = new TemplateQueryBuilder(new Template(query, ScriptType.INLINE, "mockscript",
+            XContentType.JSON, Collections.emptyMap())).queryName("outer");
+        assertEquals(new BoolQueryBuilder().must(new MatchAllQueryBuilder().queryName("foobar")).queryName("outer"),
+            builder.rewrite(queryShardContext()));
+    }
+
+    public void testRewriteWithInnerBoost() throws IOException {
+        final TermQueryBuilder query = new TermQueryBuilder("foo", "bar").boost(2);
+        QueryBuilder<?> builder = new TemplateQueryBuilder(new Template(query.toString(), ScriptType.INLINE, "mockscript",
+            XContentType.JSON, Collections.emptyMap()));
+        assertEquals(query, builder.rewrite(queryShardContext()));
+
+        builder = new TemplateQueryBuilder(new Template(query.toString(), ScriptType.INLINE, "mockscript",
+            XContentType.JSON, Collections.emptyMap())).boost(3);
+        assertEquals(new BoolQueryBuilder().must(query).boost(3), builder.rewrite(queryShardContext()));
+    }
+
+    @Override
+    protected Query rewrite(Query query) throws IOException {
+        // TemplateQueryBuilder adds some optimization if the template and query builder have boosts / query names that wraps
+        // the actual QueryBuilder that comes from the template into a BooleanQueryBuilder to give it an outer boost / name
+        // this causes some queries to be not exactly equal but equivalent such that we need to rewrite them before comparing.
+        if (query != null) {
+            MemoryIndex idx = new MemoryIndex();
+            return idx.createSearcher().rewrite(query);
+        }
+        return new MatchAllDocsQuery(); // null == *:*
+    }
 }

core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java

@@ -41,6 +41,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -272,5 +273,18 @@ public class TermsQueryBuilderTests extends AbstractQueryTestCase<TermsQueryBuil
 
         assertEquals(json, 2, parsed.values().size());
     }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        TermsQueryBuilder termsQueryBuilder = new TermsQueryBuilder(STRING_FIELD_NAME, randomTermsLookup());
+        try {
+            termsQueryBuilder.toQuery(queryShardContext());
+            fail();
+        } catch (UnsupportedOperationException ex) {
+            assertEquals("query must be rewritten first", ex.getMessage());
+        }
+        assertEquals(termsQueryBuilder.rewrite(queryShardContext()), new TermsQueryBuilder(STRING_FIELD_NAME,
+            randomTerms.stream().filter(x -> x != null).collect(Collectors.toList()))); // terms lookup removes null values
+    }
 }
 

core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java

@@ -19,19 +19,17 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 
-import static org.hamcrest.Matchers.equalTo;
-
 public class WrapperQueryBuilderTests extends AbstractQueryTestCase<WrapperQueryBuilder> {
 
     @Override
@@ -56,13 +54,9 @@ public class WrapperQueryBuilderTests extends AbstractQueryTestCase<WrapperQuery
 
     @Override
     protected void doAssertLuceneQuery(WrapperQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        try (XContentParser qSourceParser = XContentFactory.xContent(queryBuilder.source()).createParser(queryBuilder.source())) {
+        QueryBuilder<?> innerQuery = queryBuilder.rewrite(queryShardContext());
-            final QueryShardContext contextCopy = new QueryShardContext(context);
+        Query expected = rewrite(innerQuery.toQuery(context));
-            contextCopy.reset(qSourceParser);
+        assertEquals(rewrite(query), expected);
-            QueryBuilder<?> innerQuery = contextCopy.parseContext().parseInnerQueryBuilder();
-            Query expected = innerQuery.toQuery(context);
-            assertThat(query, equalTo(expected));
-        }
     }
 
     public void testIllegalArgument() {
@@ -133,4 +127,47 @@ public class WrapperQueryBuilderTests extends AbstractQueryTestCase<WrapperQuery
             throw new RuntimeException(e);
         }
     }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        TermQueryBuilder tqb = new TermQueryBuilder("foo", "bar");
+        WrapperQueryBuilder qb = new WrapperQueryBuilder(tqb.toString());
+        try {
+            qb.toQuery(queryShardContext());
+            fail();
+        } catch (UnsupportedOperationException e) {
+            assertEquals("this query must be rewritten first", e.getMessage());
+        }
+        QueryBuilder<?> rewrite = qb.rewrite(queryShardContext());
+        assertEquals(tqb, rewrite);
+    }
+
+    public void testRewriteWithInnerName() throws IOException {
+        QueryBuilder<?> builder = new WrapperQueryBuilder("{ \"match_all\" : {\"_name\" : \"foobar\"}}");
+        assertEquals(new MatchAllQueryBuilder().queryName("foobar"), builder.rewrite(queryShardContext()));
+        builder = new WrapperQueryBuilder("{ \"match_all\" : {\"_name\" : \"foobar\"}}").queryName("outer");
+        assertEquals(new BoolQueryBuilder().must(new MatchAllQueryBuilder().queryName("foobar")).queryName("outer"),
+            builder.rewrite(queryShardContext()));
+    }
+
+    public void testRewriteWithInnerBoost() throws IOException {
+        final TermQueryBuilder query = new TermQueryBuilder("foo", "bar").boost(2);
+        QueryBuilder<?> builder = new WrapperQueryBuilder(query.toString());
+        assertEquals(query, builder.rewrite(queryShardContext()));
+        builder = new WrapperQueryBuilder(query.toString()).boost(3);
+        assertEquals(new BoolQueryBuilder().must(query).boost(3), builder.rewrite(queryShardContext()));
+    }
+
+    @Override
+    protected Query rewrite(Query query) throws IOException {
+        // WrapperQueryBuilder adds some optimization if the wrapper and query builder have boosts / query names that wraps
+        // the actual QueryBuilder that comes from the binary blob into a BooleanQueryBuilder to give it an outer boost / name
+        // this causes some queries to be not exactly equal but equivalent such that we need to rewrite them before comparing.
+        if (query != null) {
+            MemoryIndex idx = new MemoryIndex();
+            return idx.createSearcher().rewrite(query);
+        }
+        return new MatchAllDocsQuery(); // null == *:*
+    }
+
 }

core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java

@@ -39,6 +39,7 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.RandomQueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.WrapperQueryBuilder;
 import org.elasticsearch.index.query.functionscore.exp.ExponentialDecayFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.fieldvaluefactor.FieldValueFactorFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.gauss.GaussDecayFunctionBuilder;
@@ -643,6 +644,35 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
         assertEquals(json, 1, parsed.getMinScore(), 0.0001);
     }
 
+    @Override
+    public void testMustRewrite() throws IOException {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        super.testMustRewrite();
+    }
+
+    public void testRewrite() throws IOException {
+        FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString()));
+        FunctionScoreQueryBuilder rewrite = (FunctionScoreQueryBuilder) functionScoreQueryBuilder.rewrite(queryShardContext());
+        assertNotSame(functionScoreQueryBuilder, rewrite);
+        assertEquals(rewrite.query(), new TermQueryBuilder("foo", "bar"));
+    }
+
+    public void testRewriteWithFunction() throws IOException {
+        TermQueryBuilder secondFunction = new TermQueryBuilder("tq", "2");
+        QueryBuilder queryBuilder = randomBoolean() ? new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString()) : new TermQueryBuilder("foo", "bar");
+        FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder,
+            new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{
+                new FunctionScoreQueryBuilder.FilterFunctionBuilder(new WrapperQueryBuilder(new TermQueryBuilder("tq", "1").toString()), new RandomScoreFunctionBuilder()),
+                new FunctionScoreQueryBuilder.FilterFunctionBuilder(secondFunction, new RandomScoreFunctionBuilder())
+
+            });
+        FunctionScoreQueryBuilder rewrite = (FunctionScoreQueryBuilder) functionScoreQueryBuilder.rewrite(queryShardContext());
+        assertNotSame(functionScoreQueryBuilder, rewrite);
+        assertEquals(rewrite.query(), new TermQueryBuilder("foo", "bar"));
+        assertEquals(rewrite.filterFunctionBuilders()[0].getFilter(), new TermQueryBuilder("tq", "1"));
+        assertSame(rewrite.filterFunctionBuilders()[1].getFilter(), secondFunction);
+    }
+
     public void testQueryMalformedArrayNotSupported() throws IOException {
         String json =
             "{\n" +

core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java

@@ -84,7 +84,7 @@ public class PercolateDocumentParserTests extends ESTestCase {
         Map<String, QueryParser<?>> parsers = singletonMap("term", new TermQueryParser());
         IndicesQueriesRegistry indicesQueriesRegistry = new IndicesQueriesRegistry(indexSettings.getSettings(), parsers);
 
-        queryShardContext = new QueryShardContext(indexSettings, null, null, null, mapperService, null, null, indicesQueriesRegistry);
+        queryShardContext = new QueryShardContext(indexSettings, null, null, mapperService, null, null, indicesQueriesRegistry);
 
         HighlightPhase highlightPhase = new HighlightPhase(Settings.EMPTY, new Highlighters());
         AggregatorParsers aggregatorParsers = new AggregatorParsers(Collections.emptySet(), Collections.emptySet());

core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java

@@ -41,9 +41,14 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.query.AbstractQueryTestCase;
+import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.EmptyQueryBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.WrapperQueryBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionParser;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.Script;
@@ -483,4 +488,14 @@ public class SearchSourceBuilderTests extends ESTestCase {
         String query = "{ \"post_filter\": {} }";
         assertParseSearchSource(builder, new BytesArray(query));
     }
+
+    public void testRewrite() throws IOException {
+        SearchSourceBuilder builder = new SearchSourceBuilder();
+        builder.query(new BoolQueryBuilder());
+        TermQueryBuilder tqb = new TermQueryBuilder("foo", "bar");
+        builder.postFilter(new WrapperQueryBuilder(tqb.toString()));
+        builder.rewrite(new QueryRewriteContext(null, null, indicesQueriesRegistry));
+        assertEquals(new MatchAllQueryBuilder(), builder.query());
+        assertEquals(tqb, builder.postFilter());
+    }
 }

core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java

@@ -277,7 +277,7 @@ public class HighlightBuilderTests extends ESTestCase {
         Index index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter
-        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, null, indicesQueriesRegistry) {
+        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 StringFieldMapper.Builder builder = new StringFieldMapper.Builder(name);

core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java

@@ -159,7 +159,7 @@ public class QueryRescoreBuilderTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer
-        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, null, indicesQueriesRegistry) {
+        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 StringFieldMapper.Builder builder = new StringFieldMapper.Builder(name);
@@ -169,7 +169,7 @@ public class QueryRescoreBuilderTests extends ESTestCase {
 
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
             RescoreBuilder<?> rescoreBuilder = randomRescoreBuilder();
-            QueryRescoreContext rescoreContext = (QueryRescoreContext) rescoreBuilder.build(mockShardContext);
+            QueryRescoreContext rescoreContext = rescoreBuilder.build(mockShardContext);
             XContentParser parser = createParser(rescoreBuilder);
 
             QueryRescoreContext parsedRescoreContext = (QueryRescoreContext) new RescoreParseElement().parseSingleRescoreContext(parser, mockShardContext);

core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java

@@ -170,7 +170,7 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
             }
         };
 
-        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, mockMapperService, null, null, null) {
+        QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, mockMapperService, null, null, null) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 StringFieldMapper.Builder builder = new StringFieldMapper.Builder(name);

modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java

@@ -46,6 +46,7 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.TemplateQueryParser;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionParser;
@@ -152,7 +153,7 @@ public class TemplateQueryParserTests extends ESTestCase {
             }
         });
         IndicesQueriesRegistry indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
-        context = new QueryShardContext(idxSettings, proxy, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry);
+        context = new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry);
     }
 
     @Override
@@ -170,7 +171,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         templateSourceParser.nextToken();
 
         TemplateQueryParser parser = injector.getInstance(TemplateQueryParser.class);
-        Query query = parser.fromXContent(context.parseContext()).toQuery(context);
+        Query query = QueryBuilder.rewriteQuery(parser.fromXContent(context.parseContext()), context).toQuery(context);
         assertTrue("Parsing template query failed.", query instanceof MatchAllDocsQuery);
     }
 
@@ -181,7 +182,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         context.reset(templateSourceParser);
 
         TemplateQueryParser parser = injector.getInstance(TemplateQueryParser.class);
-        Query query = parser.fromXContent(context.parseContext()).toQuery(context);
+        Query query = QueryBuilder.rewriteQuery(parser.fromXContent(context.parseContext()), context).toQuery(context);
         assertTrue("Parsing template query failed.", query instanceof MatchAllDocsQuery);
     }
 
@@ -199,7 +200,7 @@ public class TemplateQueryParserTests extends ESTestCase {
 
         TemplateQueryParser parser = injector.getInstance(TemplateQueryParser.class);
         try {
-            parser.fromXContent(context.parseContext()).toQuery(context);
+            parser.fromXContent(context.parseContext()).rewrite(context);
             fail("Expected ParsingException");
         } catch (ParsingException e) {
             assertThat(e.getMessage(), containsString("query malformed, no field after start_object"));
@@ -213,8 +214,24 @@ public class TemplateQueryParserTests extends ESTestCase {
         context.reset(templateSourceParser);
         templateSourceParser.nextToken();
 
+
         TemplateQueryParser parser = injector.getInstance(TemplateQueryParser.class);
-        Query query = parser.fromXContent(context.parseContext()).toQuery(context);
+        Query query = QueryBuilder.rewriteQuery(parser.fromXContent(context.parseContext()), context).toQuery(context);
         assertTrue("Parsing template query failed.", query instanceof MatchAllDocsQuery);
     }
+
+    public void testMustRewrite() throws Exception {
+        String templateString = "{ \"file\": \"storedTemplate\" ,\"params\":{\"template\":\"all\" } } ";
+
+        XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString);
+        context.reset(templateSourceParser);
+        templateSourceParser.nextToken();
+        TemplateQueryParser parser = injector.getInstance(TemplateQueryParser.class);
+        try {
+            parser.fromXContent(context.parseContext()).toQuery(context);
+            fail();
+        } catch (UnsupportedOperationException ex) {
+            assertEquals("this query must be rewritten first", ex.getMessage());
+        }
+    }
 }