updates to fix some of the NOCOMMITs

core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java

@@ -28,6 +28,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.warmer.IndexWarmersMetaData;
 
 import java.io.IOException;
@@ -122,7 +123,7 @@ public class GetIndexResponse extends ActionResponse {
                         in.readString(),
                         in.readStringArray(),
                         in.readOptionalBoolean(),
-                        in.readBytesReference())
+                        SearchSourceBuilder.PROTOTYPE.readFrom(in))
                 );
             }
             warmersMapBuilder.put(key, Collections.unmodifiableList(warmerEntryBuilder));
@@ -173,7 +174,7 @@ public class GetIndexResponse extends ActionResponse {
                 out.writeString(warmerEntry.name());
                 out.writeStringArray(warmerEntry.types());
                 out.writeOptionalBoolean(warmerEntry.requestCache());
-                out.writeBytesReference(warmerEntry.source());
+                warmerEntry.source().writeTo(out);
             }
         }
         out.writeVInt(mappings.size());

core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java

@@ -20,12 +20,12 @@
 package org.elasticsearch.action.admin.indices.warmer.get;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-import org.elasticsearch.Version;
+
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.warmer.IndexWarmersMetaData;
 
 import java.io.IOException;
@@ -69,7 +69,7 @@ public class GetWarmersResponse extends ActionResponse {
             for (int j = 0; j < valueSize; j++) {
                 String name = in.readString();
                 String[] types = in.readStringArray();
-                BytesReference source = in.readBytesReference();
+                SearchSourceBuilder source = SearchSourceBuilder.PROTOTYPE.readFrom(in);
                 Boolean queryCache = null;
                 queryCache = in.readOptionalBoolean();
                 warmerEntryBuilder.add(new IndexWarmersMetaData.Entry(
@@ -94,7 +94,7 @@ public class GetWarmersResponse extends ActionResponse {
             for (IndexWarmersMetaData.Entry warmerEntry : indexEntry.value) {
                 out.writeString(warmerEntry.name());
                 out.writeStringArray(warmerEntry.types());
-                out.writeBytesReference(warmerEntry.source());
+                warmerEntry.source().writeTo(out);
                 out.writeOptionalBoolean(warmerEntry.requestCache());
             }
         }

core/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java

@@ -38,6 +38,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.warmer.IndexWarmersMetaData;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -114,11 +115,9 @@ public class TransportPutWarmerAction extends TransportMasterNodeAction<PutWarme
                         MetaData metaData = currentState.metaData();
                         String[] concreteIndices = indexNameExpressionResolver.concreteIndices(currentState, request.searchRequest().indicesOptions(), request.searchRequest().indices());
 
-                        BytesReference source = null;
+                        SearchSourceBuilder source = null;
                         if (request.searchRequest().source() != null) {
-//                            source = request.searchRequest().source(); // NOCOMMIT fix this
+                            source = request.searchRequest().source();
-                        } else if (request.searchRequest().extraSource() != null && request.searchRequest().extraSource().length() > 0) {
-                            source = request.searchRequest().extraSource();
                         }
 
                         // now replace it on the metadata

core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -21,6 +21,7 @@ package org.elasticsearch.cluster.metadata;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.Diff;
@@ -40,7 +41,11 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
-import org.elasticsearch.common.xcontent.*;
+import org.elasticsearch.common.xcontent.FromXContentBuilder;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.warmer.IndexWarmersMetaData;
@@ -56,12 +61,14 @@ import java.util.Map;
 
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR;
-import static org.elasticsearch.common.settings.Settings.*;
+import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
 
 /**
  *
  */
-public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuilder<IndexMetaData>, ToXContent  {
+public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuilder<IndexMetaData>, ToXContent {
 
     public static final IndexMetaData PROTO = IndexMetaData.builder("")
             .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
@@ -524,7 +531,8 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     }
 
     @Override
-    public IndexMetaData fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public IndexMetaData fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher)
+            throws IOException {
         return Builder.fromXContent(parser);
     }
 
@@ -705,7 +713,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         public int numberOfReplicas() {
             return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1);
         }
-        
+
         public Builder creationDate(long creationDate) {
             settings = settingsBuilder().put(settings).put(SETTING_CREATION_DATE, creationDate).build();
             return this;

core/src/main/java/org/elasticsearch/index/search/stats/SearchSlowLog.java

@@ -190,13 +190,7 @@ public final class SearchSlowLog{
             }
             sb.append("search_type[").append(context.searchType()).append("], total_shards[").append(context.numberOfShards()).append("], ");
             if (context.request().source() != null) {
-                // try {
+                sb.append("source[").append(context.request().source()).append("], ");
-                // //
-                // sb.append("source[").append(XContentHelper.convertToJson(context.request().source(),
-                // // reformat)).append("], "); NOCOMMIT fix this
-                // } catch (IOException e) {
-                // sb.append("source[_failed_to_convert_], ");
-                // }
             } else {
                 sb.append("source[], ");
             }

core/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java

@@ -19,14 +19,27 @@
 package org.elasticsearch.rest.action.admin.indices.warmer.put;
 
 import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.support.AcknowledgedRestListener;
+import org.elasticsearch.rest.action.support.RestActions;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
@@ -35,9 +48,12 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT;
  */
 public class RestPutWarmerAction extends BaseRestHandler {
 
+    private final IndicesQueriesRegistry queryRegistry;
+
     @Inject
-    public RestPutWarmerAction(Settings settings, RestController controller, Client client) {
+    public RestPutWarmerAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry) {
         super(settings, controller, client);
+        this.queryRegistry = queryRegistry;
         controller.registerHandler(PUT, "/_warmer/{name}", this);
         controller.registerHandler(PUT, "/{index}/_warmer/{name}", this);
         controller.registerHandler(PUT, "/{index}/{type}/_warmer/{name}", this);
@@ -56,16 +72,19 @@ public class RestPutWarmerAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException {
         PutWarmerRequest putWarmerRequest = new PutWarmerRequest(request.param("name"));
-        // SearchRequest searchRequest = new
+
-        // SearchRequest(Strings.splitStringByCommaToArray(request.param("index")))
+        BytesReference sourceBytes = RestActions.getRestContent(request);
-        // .types(Strings.splitStringByCommaToArray(request.param("type")))
+        XContentParser parser = XContentFactory.xContent(sourceBytes).createParser(sourceBytes);
-        // .requestCache(request.paramAsBoolean("request_cache", null))
+        QueryParseContext queryParseContext = new QueryParseContext(new Index(""), queryRegistry); // NORELEASE remove index
-        // .source(request.content());
+        queryParseContext.reset(parser);
-        // searchRequest.indicesOptions(IndicesOptions.fromRequest(request,
+        SearchSourceBuilder source = SearchSourceBuilder.PROTOTYPE.fromXContent(parser, queryParseContext);
-        // searchRequest.indicesOptions()));
+        SearchRequest searchRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")))
-        // putWarmerRequest.searchRequest(searchRequest); NOCOMMIT fix this
+                .types(Strings.splitStringByCommaToArray(request.param("type")))
+                .requestCache(request.paramAsBoolean("request_cache", null)).source(source);
+        searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
+        putWarmerRequest.searchRequest(searchRequest);
         putWarmerRequest.timeout(request.paramAsTime("timeout", putWarmerRequest.timeout()));
         putWarmerRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putWarmerRequest.masterNodeTimeout()));
         client.admin().indices().putWarmer(putWarmerRequest, new AcknowledgedRestListener<>(channel));

core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java

@@ -26,9 +26,15 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
@@ -43,6 +49,8 @@ import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.search.suggest.SuggestBuilder;
 
+import java.io.IOException;
+
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
@@ -53,9 +61,12 @@ import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
  */
 public class RestSearchAction extends BaseRestHandler {
 
+    private final IndicesQueriesRegistry queryRegistry;
+
     @Inject
-    public RestSearchAction(Settings settings, RestController controller, Client client) {
+    public RestSearchAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry) {
         super(settings, controller, client);
+        this.queryRegistry = queryRegistry;
         controller.registerHandler(GET, "/_search", this);
         controller.registerHandler(POST, "/_search", this);
         controller.registerHandler(GET, "/{index}/_search", this);
@@ -79,13 +90,13 @@ public class RestSearchAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException {
         SearchRequest searchRequest;
-        searchRequest = RestSearchAction.parseSearchRequest(request, parseFieldMatcher);
+        searchRequest = RestSearchAction.parseSearchRequest(request, parseFieldMatcher, queryRegistry);
         client.search(searchRequest, new RestStatusToXContentListener<SearchResponse>(channel));
     }
 
-    public static SearchRequest parseSearchRequest(RestRequest request, ParseFieldMatcher parseFieldMatcher) {
+    public static SearchRequest parseSearchRequest(RestRequest request, ParseFieldMatcher parseFieldMatcher, IndicesQueriesRegistry queryRegistry) throws IOException {
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         SearchRequest searchRequest = new SearchRequest(indices);
         // get the content, and put it in the body
@@ -95,8 +106,11 @@ public class RestSearchAction extends BaseRestHandler {
             if (isTemplateRequest) {
                 searchRequest.templateSource(RestActions.getRestContent(request));
             } else {
-                // searchRequest.source(RestActions.getRestContent(request));
+                 BytesReference sourceBytes = RestActions.getRestContent(request);
-                // NOCOMMIT fix this
+                 XContentParser parser = XContentFactory.xContent(sourceBytes).createParser(sourceBytes);
+                 QueryParseContext queryParseContext = new QueryParseContext(new Index(""), queryRegistry); // NORELEASE remove index
+                 queryParseContext.reset(parser);
+                 searchRequest.source(SearchSourceBuilder.PROTOTYPE.fromXContent(parser, queryParseContext));
             }
         }
 

core/src/main/java/org/elasticsearch/search/SearchService.java

@@ -30,6 +30,7 @@ import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -47,6 +48,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
@@ -76,6 +78,7 @@ import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script.ScriptParseException;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -86,11 +89,16 @@ import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.QueryFetchSearchResult;
 import org.elasticsearch.search.fetch.ScrollQueryFetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchRequest;
+import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext;
+import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext.FieldDataField;
+import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
+import org.elasticsearch.search.fetch.script.ScriptFieldsContext.ScriptField;
 import org.elasticsearch.search.internal.DefaultSearchContext;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContext.Lifetime;
+import org.elasticsearch.search.internal.ShardSearchLocalRequest;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QueryPhase;
 import org.elasticsearch.search.query.QuerySearchRequest;
@@ -101,6 +109,7 @@ import org.elasticsearch.search.warmer.IndexWarmersMetaData;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
@@ -748,54 +757,65 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
         context.aggregations(null); // NOCOMMIT parse source.aggregations()
                                     // ByteReference into
                                     // SearchContextAggregations object
-        context.suggest(null); // NOCOMMIT parse source.suggest() ByteReference
+        XContentParser suggestParser = null;
-                               // into SuggestionSearchContext object
+        try {
+            suggestParser = XContentFactory.xContent(source.suggest()).createParser(source.suggest());
+            this.elementParsers.get("suggest").parse(suggestParser, context);
+        } catch (Exception e) {
+            String sSource = "_na_";
+            try {
+                sSource = source.toString();
+            } catch (Throwable e1) {
+                // ignore
+            }
+            XContentLocation location = suggestParser != null ? suggestParser.getTokenLocation() : null;
+            throw new SearchParseException(context, "failed to parse suggest source [" + sSource + "]", location, e);
+        }
         context.addRescore(null);// NOCOMMIT parse source.rescore()
                                  // ByteReference into RescoreSearchContext
                                  // object
-        // NOCOMMIT populate the rest of the search request
+        context.fieldNames().addAll(source.fields());
         context.explain(source.explain());
+        context.fetchSourceContext(source.fetchSource());
+        FieldDataFieldsContext fieldDataFieldsContext = context.getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY);
+        for (String field : source.fieldDataFields()) {
+            fieldDataFieldsContext.add(new FieldDataField(field));
+        }
+        XContentParser highlighterParser = null;
+        try {
+            highlighterParser = XContentFactory.xContent(source.highlighter()).createParser(source.highlighter());
+            this.elementParsers.get("highlight").parse(highlighterParser, context);
+        } catch (Exception e) {
+            String sSource = "_na_";
+            try {
+                sSource = source.toString();
+            } catch (Throwable e1) {
+                // ignore
+            }
+            XContentLocation location = highlighterParser != null ? highlighterParser.getTokenLocation() : null;
+            throw new SearchParseException(context, "failed to parse suggest source [" + sSource + "]", location, e);
+        }
+        XContentParser innerHitsParser = null;
+        try {
+            innerHitsParser = XContentFactory.xContent(source.innerHits()).createParser(source.innerHits());
+            this.elementParsers.get("highlight").parse(innerHitsParser, context);
+        } catch (Exception e) {
+            String sSource = "_na_";
+            try {
+                sSource = source.toString();
+            } catch (Throwable e1) {
+                // ignore
+            }
+            XContentLocation location = innerHitsParser != null ? innerHitsParser.getTokenLocation() : null;
+            throw new SearchParseException(context, "failed to parse suggest source [" + sSource + "]", location, e);
+        }
+        for (org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField field : source.scriptFields()) {
+            SearchScript searchScript = context.scriptService().search(context.lookup(), field.script(), ScriptContext.Standard.SEARCH);
+            context.scriptFields().add(new ScriptField(field.fieldName(), searchScript, false)); // NORELEASE need to have ignore_exception parsed somewhere
+        }
+        // NOCOMMIT need to work out what to do about term_vectors_fetch (previously handled by TermVectorsFetchParseElement) as this is not available as an option in SearchSourceBuilder
         context.version(source.version());
-
+        context.groupStats(Arrays.asList(source.stats())); // NORELEASE stats should be a list in SearchSourceBuilder
-//        XContentParser parser = null;
-//        try {
-//            parser = XContentFactory.xContent(source).createParser(source);
-//            XContentParser.Token token;
-//            token = parser.nextToken();
-//            if (token != XContentParser.Token.START_OBJECT) {
-//                throw new ElasticsearchParseException("failed to parse search source. source must be an object, but found [{}] instead", token.name());
-//            }
-//            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-//                if (token == XContentParser.Token.FIELD_NAME) {
-//                    String fieldName = parser.currentName();
-//                    parser.nextToken();
-//                    SearchParseElement element = elementParsers.get(fieldName);
-//                    if (element == null) {
-//                        throw new SearchParseException(context, "failed to parse search source. unknown search element [" + fieldName + "]", parser.getTokenLocation());
-//                    }
-//                    element.parse(parser, context);
-//                } else {
-//                    if (token == null) {
-//                        throw new ElasticsearchParseException("failed to parse search source. end of query source reached but query is not complete.");
-//                    } else {
-//                        throw new ElasticsearchParseException("failed to parse search source. expected field name but got [{}]", token);
-//                    }
-//                }
-//            }
-//        } catch (Throwable e) {
-//            String sSource = "_na_";
-//            try {
-//                sSource = XContentHelper.convertToJson(source, false);
-//            } catch (Throwable e1) {
-//                // ignore
-//            }
-//            XContentLocation location = parser != null ? parser.getTokenLocation() : null;
-//            throw new SearchParseException(context, "failed to parse search source [" + sSource + "]", location, e);
-//        } finally {
-//            if (parser != null) {
-//                parser.close();
-//            }
-//        }
     }
 
     private static final int[] EMPTY_DOC_IDS = new int[0];
@@ -1088,32 +1108,26 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
                         SearchContext context = null;
                         try {
                             long now = System.nanoTime();
-                            // ShardSearchRequest request = new
+                            ShardSearchRequest request = new ShardSearchLocalRequest(indexShard.shardId(), indexMetaData.numberOfShards(),
-                            // ShardSearchLocalRequest(indexShard.shardId(),
+                                    SearchType.QUERY_THEN_FETCH, entry.source(), entry.types(), entry.requestCache());
-                            // indexMetaData.numberOfShards(),
+                            context = createContext(request, warmerContext.searcher());
-                            // SearchType.QUERY_THEN_FETCH, entry.source(),
+                            // if we use sort, we need to do query to sort on
-                            // entry.types(), entry.requestCache());
-                            // context = createContext(request,
-                            // warmerContext.searcher());
-                            // // if we use sort, we need to do query to sort on
                             // it and load relevant field data
-                            // // if not, we might as well set size=0 (and cache
+                            // if not, we might as well set size=0 (and cache
                             // if needed)
-                            // if (context.sort() == null) {
+                            if (context.sort() == null) {
-                            // context.size(0);
+                                context.size(0);
-                            // }
+                            }
-                            // boolean canCache =
+                            boolean canCache = indicesQueryCache.canCache(request, context);
-                            // indicesQueryCache.canCache(request, context);
+                            // early terminate when we can cache, since we
-                            // // early terminate when we can cache, since we
                             // can only do proper caching on top level searcher
-                            // // also, if we can't cache, and its top, we don't
+                            // also, if we can't cache, and its top, we don't
                             // need to execute it, since we already did when its
                             // not top
-                            // if (canCache != top) {
+                            if (canCache != top) {
-                            // return;
+                                return;
-                            // }
+                            }
-                            // loadOrExecuteQueryPhase(request, context,
+                            loadOrExecuteQueryPhase(request, context, queryPhase);
-                            // queryPhase); NOCOMMIT fix this
                             long took = System.nanoTime() - now;
                             if (indexShard.warmerService().logger().isTraceEnabled()) {
                                 indexShard.warmerService().logger().trace("warmed [{}], took [{}]", entry.name(), TimeValue.timeValueNanos(took));

core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java

@@ -1022,7 +1022,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         }
     }
 
-    private static class ScriptField implements Writeable<ScriptField>, ToXContent {
+    public static class ScriptField implements Writeable<ScriptField>, ToXContent {
 
         public static final ScriptField PROTOTYPE = new ScriptField(null, null);
 

core/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java

@@ -23,8 +23,6 @@ import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -32,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -67,10 +66,10 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
     public static class Entry {
         private final String name;
         private final String[] types;
-        private final BytesReference source;
+        private final SearchSourceBuilder source;
         private final Boolean requestCache;
 
-        public Entry(String name, String[] types, Boolean requestCache, BytesReference source) {
+        public Entry(String name, String[] types, Boolean requestCache, SearchSourceBuilder source) {
             this.name = name;
             this.types = types == null ? Strings.EMPTY_ARRAY : types;
             this.source = source;
@@ -86,7 +85,7 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
         }
 
         @Nullable
-        public BytesReference source() {
+        public SearchSourceBuilder source() {
             return this.source;
         }
 
@@ -141,9 +140,9 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
         for (int i = 0; i < entries.length; i++) {
             String name = in.readString();
             String[] types = in.readStringArray();
-            BytesReference source = null;
+            SearchSourceBuilder source = null;
             if (in.readBoolean()) {
-                source = in.readBytesReference();
+                source = SearchSourceBuilder.PROTOTYPE.readFrom(in);
             }
             Boolean queryCache;
             queryCache = in.readOptionalBoolean();
@@ -162,7 +161,7 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
                 out.writeBoolean(false);
             } else {
                 out.writeBoolean(true);
-                out.writeBytesReference(entry.source());
+                entry.source.writeTo(out);
             }
             out.writeOptionalBoolean(entry.requestCache());
         }
@@ -194,7 +193,7 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
             } else if (token == XContentParser.Token.START_OBJECT) {
                 String name = currentFieldName;
                 List<String> types = new ArrayList<>(2);
-                BytesReference source = null;
+                SearchSourceBuilder source = null;
                 Boolean queryCache = null;
                 while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                     if (token == XContentParser.Token.FIELD_NAME) {
@@ -207,13 +206,12 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
                         }
                     } else if (token == XContentParser.Token.START_OBJECT) {
                         if ("source".equals(currentFieldName)) {
-                            XContentBuilder builder = XContentFactory.jsonBuilder().map(parser.mapOrdered());
+                            source = SearchSourceBuilder.PROTOTYPE.fromXContent(parser, null); // NOCOMMIT need context from somewhere
-                            source = builder.bytes();
-                        }
-                    } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
-                        if ("source".equals(currentFieldName)) {
-                            source = new BytesArray(parser.binaryValue());
                         }
+//                    } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
+//                        if ("source".equals(currentFieldName)) {
+//                            source = new BytesArray(parser.binaryValue());
+//                        } NORELEASE do we need this?
                     } else if (token.isValue()) {
                         if ("requestCache".equals(currentFieldName) || "request_cache".equals(currentFieldName)) {
                             queryCache = parser.booleanValue();
@@ -239,22 +237,12 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
     }
 
     public static void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException {
-        boolean binary = params.paramAsBoolean("binary", false);
         builder.startObject(entry.name(), XContentBuilder.FieldCaseConversion.NONE);
         builder.field("types", entry.types());
         if (entry.requestCache() != null) {
             builder.field("requestCache", entry.requestCache());
         }
-        builder.field("source");
+        builder.field("source", entry.source());
-        if (binary) {
-            builder.value(entry.source());
-        } else {
-            Map<String, Object> mapping;
-            try (XContentParser parser = XContentFactory.xContent(entry.source()).createParser(entry.source())) {
-                mapping = parser.mapOrdered();
-            }
-            builder.map(mapping);
-        }
         builder.endObject();
     }
 

core/src/test/java/org/elasticsearch/action/count/CountRequestTests.java

@@ -31,6 +31,7 @@ import org.junit.Test;
 import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.CoreMatchers.nullValue;
 
 public class CountRequestTests extends ESTestCase {
@@ -75,11 +76,7 @@ public class CountRequestTests extends ESTestCase {
         if (countRequest.source() == null) {
             assertThat(searchRequest.source(), nullValue());
         } else {
-            // Map<String, Object> sourceMap =
+            assertThat(searchRequest.source().query(), notNullValue());
-            // XContentHelper.convertToMap(searchRequest.source(), false).v2();
-            // assertThat(sourceMap.size(), equalTo(1));
-            // assertThat(sourceMap.get("query"), notNullValue()); NOCOMMIT fix
-            // this
         }
 
         Map<String, Object> extraSourceMap = XContentHelper.convertToMap(searchRequest.extraSource(), false).v2();

core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java

@@ -21,6 +21,7 @@ package org.elasticsearch.cluster;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.google.common.collect.ImmutableMap;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -39,6 +40,7 @@ import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.warmer.IndexWarmersMetaData;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.Test;
@@ -531,7 +533,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
                             randomName("warm"),
                             new String[]{randomName("type")},
                             randomBoolean(),
-                            new BytesArray(randomAsciiOfLength(1000)))
+                            new SearchSourceBuilder()) // NOCOMMIT this used to be new BytesArray(randomAsciiOfLength(1000)) whiat should it be now?
             );
         } else {
             return new IndexWarmersMetaData();

core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java

@@ -48,10 +48,10 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.suggest.SuggestRequestBuilder;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.suggest.SuggestBuilders;
 import org.elasticsearch.search.warmer.IndexWarmersMetaData;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -61,7 +61,9 @@ import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuild
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.*;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
 
@@ -508,7 +510,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())
                 .execute().actionGet();
         assertHitCount(response, 0l);
-        
+
         //you should still be able to run empty searches without things blowing up
         response  = client().prepareSearch()
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())
@@ -613,7 +615,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         assertThat(client().admin().indices().prepareExists("bar").get().isExists(), equalTo(false));
         assertThat(client().admin().indices().prepareExists("barbaz").get().isExists(), equalTo(false));
     }
-    
+
     @Test
     public void testPutWarmer() throws Exception {
         createIndex("foobar");
@@ -622,26 +624,26 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         assertThat(client().admin().indices().prepareGetWarmers("foobar").setWarmers("warmer1").get().getWarmers().size(), equalTo(1));
 
     }
-    
+
     @Test
     public void testPutWarmer_wildcard() throws Exception {
         createIndex("foo", "foobar", "bar", "barbaz");
         ensureYellow();
 
         verify(client().admin().indices().preparePutWarmer("warmer1").setSearchRequest(client().prepareSearch().setIndices("foo*").setQuery(QueryBuilders.matchAllQuery())), false);
-        
+
         assertThat(client().admin().indices().prepareGetWarmers("foo").setWarmers("warmer1").get().getWarmers().size(), equalTo(1));
         assertThat(client().admin().indices().prepareGetWarmers("foobar").setWarmers("warmer1").get().getWarmers().size(), equalTo(1));
         assertThat(client().admin().indices().prepareGetWarmers("bar").setWarmers("warmer1").get().getWarmers().size(), equalTo(0));
         assertThat(client().admin().indices().prepareGetWarmers("barbaz").setWarmers("warmer1").get().getWarmers().size(), equalTo(0));
 
         verify(client().admin().indices().preparePutWarmer("warmer2").setSearchRequest(client().prepareSearch().setIndices().setQuery(QueryBuilders.matchAllQuery())), false);
-        
+
         assertThat(client().admin().indices().prepareGetWarmers("foo").setWarmers("warmer2").get().getWarmers().size(), equalTo(1));
         assertThat(client().admin().indices().prepareGetWarmers("foobar").setWarmers("warmer2").get().getWarmers().size(), equalTo(1));
         assertThat(client().admin().indices().prepareGetWarmers("bar").setWarmers("warmer2").get().getWarmers().size(), equalTo(1));
         assertThat(client().admin().indices().prepareGetWarmers("barbaz").setWarmers("warmer2").get().getWarmers().size(), equalTo(1));
-        
+
     }
 
     @Test
@@ -652,7 +654,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("foobar").get().exists(), equalTo(true));
 
     }
-    
+
     @Test
     public void testPutAlias_wildcard() throws Exception {
         createIndex("foo", "foobar", "bar", "barbaz");
@@ -669,14 +671,13 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("foobar").get().exists(), equalTo(true));
         assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("bar").get().exists(), equalTo(true));
         assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("barbaz").get().exists(), equalTo(true));
-        
+
     }
-    
+
     @Test
     public void testDeleteWarmer() throws Exception {
-        IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry(
+        SearchSourceBuilder source = new SearchSourceBuilder();
-                "test1", new String[]{"typ1"}, false, new BytesArray("{\"query\" : { \"match_all\" : {}}}")
+        IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "typ1" }, false, source);
-        );
         assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry)));
         ensureYellow();
 
@@ -690,9 +691,8 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
     public void testDeleteWarmer_wildcard() throws Exception {
         verify(client().admin().indices().prepareDeleteWarmer().setIndices("_all").setNames("test1"), true);
 
-        IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry(
+        SearchSourceBuilder source = new SearchSourceBuilder();
-                "test1", new String[]{"type1"}, false, new BytesArray("{\"query\" : { \"match_all\" : {}}}")
+        IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "type1" }, false, source);
-        );
         assertAcked(prepareCreate("foo").addCustom(new IndexWarmersMetaData(entry)));
         assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry)));
         assertAcked(prepareCreate("bar").addCustom(new IndexWarmersMetaData(entry)));
@@ -737,7 +737,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         assertThat(client().admin().indices().prepareGetMappings("foobar").get().mappings().get("foobar").get("type3"), notNullValue());
         assertThat(client().admin().indices().prepareGetMappings("bar").get().mappings().get("bar").get("type3"), notNullValue());
         assertThat(client().admin().indices().prepareGetMappings("barbaz").get().mappings().get("barbaz").get("type3"), notNullValue());
-        
+
 
         verify(client().admin().indices().preparePutMapping("c*").setType("type1").setSource("field", "type=string"), true);
 
@@ -883,7 +883,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
     private static void verify(ActionRequestBuilder requestBuilder, boolean fail) {
         verify(requestBuilder, fail, 0);
     }
-    
+
     private static void verify(ActionRequestBuilder requestBuilder, boolean fail, long expectedCount) {
         if (fail) {
             if (requestBuilder instanceof MultiSearchRequestBuilder) {