From b98f7cf0232988633927deda6adf12304383d844 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 23 Sep 2015 10:42:31 +0100 Subject: [PATCH] updates to fix some of the NOCOMMITs --- .../admin/indices/get/GetIndexResponse.java | 5 +- .../warmer/get/GetWarmersResponse.java | 8 +- .../warmer/put/TransportPutWarmerAction.java | 7 +- .../cluster/metadata/IndexMetaData.java | 18 ++- .../index/search/stats/SearchSlowLog.java | 8 +- .../warmer/put/RestPutWarmerAction.java | 39 +++-- .../rest/action/search/RestSearchAction.java | 26 +++- .../elasticsearch/search/SearchService.java | 142 ++++++++++-------- .../search/builder/SearchSourceBuilder.java | 2 +- .../search/warmer/IndexWarmersMetaData.java | 40 ++--- .../action/count/CountRequestTests.java | 7 +- .../cluster/ClusterStateDiffIT.java | 4 +- .../indices/IndicesOptionsIntegrationIT.java | 38 ++--- 13 files changed, 190 insertions(+), 154 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index 0930f8f1d4e..f0cc490d044 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import java.io.IOException; @@ -122,7 +123,7 @@ public class GetIndexResponse extends ActionResponse { in.readString(), in.readStringArray(), in.readOptionalBoolean(), - in.readBytesReference()) + SearchSourceBuilder.PROTOTYPE.readFrom(in)) ); } warmersMapBuilder.put(key, Collections.unmodifiableList(warmerEntryBuilder)); @@ -173,7 +174,7 @@ public class GetIndexResponse extends ActionResponse { out.writeString(warmerEntry.name()); out.writeStringArray(warmerEntry.types()); out.writeOptionalBoolean(warmerEntry.requestCache()); - out.writeBytesReference(warmerEntry.source()); + warmerEntry.source().writeTo(out); } } out.writeVInt(mappings.size()); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java index 3ed444c88dd..45040dad1fb 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java @@ -20,12 +20,12 @@ package org.elasticsearch.action.admin.indices.warmer.get; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.elasticsearch.Version; + import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import java.io.IOException; @@ -69,7 +69,7 @@ public class GetWarmersResponse extends ActionResponse { for (int j = 0; j < valueSize; j++) { String name = in.readString(); String[] types = in.readStringArray(); - BytesReference source = in.readBytesReference(); + SearchSourceBuilder source = SearchSourceBuilder.PROTOTYPE.readFrom(in); Boolean queryCache = null; queryCache = in.readOptionalBoolean(); warmerEntryBuilder.add(new IndexWarmersMetaData.Entry( @@ -94,7 +94,7 @@ public class GetWarmersResponse extends ActionResponse { for (IndexWarmersMetaData.Entry warmerEntry : indexEntry.value) { out.writeString(warmerEntry.name()); out.writeStringArray(warmerEntry.types()); - out.writeBytesReference(warmerEntry.source()); + warmerEntry.source().writeTo(out); out.writeOptionalBoolean(warmerEntry.requestCache()); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java index f88ae356693..883b4298ba3 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -114,11 +115,9 @@ public class TransportPutWarmerAction extends TransportMasterNodeAction 0) { - source = request.searchRequest().extraSource(); + source = request.searchRequest().source(); } // now replace it on the metadata diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 6ea1d0e6e61..5d2fe4208d8 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -21,6 +21,7 @@ package org.elasticsearch.cluster.metadata; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.Diff; @@ -40,7 +41,11 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.loader.SettingsLoader; -import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.FromXContentBuilder; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.warmer.IndexWarmersMetaData; @@ -56,12 +61,14 @@ import java.util.Map; import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND; import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR; -import static org.elasticsearch.common.settings.Settings.*; +import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; /** * */ -public class IndexMetaData implements Diffable, FromXContentBuilder, ToXContent { +public class IndexMetaData implements Diffable, FromXContentBuilder, ToXContent { public static final IndexMetaData PROTO = IndexMetaData.builder("") .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) @@ -524,7 +531,8 @@ public class IndexMetaData implements Diffable, FromXContentBuild } @Override - public IndexMetaData fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException { + public IndexMetaData fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) + throws IOException { return Builder.fromXContent(parser); } @@ -705,7 +713,7 @@ public class IndexMetaData implements Diffable, FromXContentBuild public int numberOfReplicas() { return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1); } - + public Builder creationDate(long creationDate) { settings = settingsBuilder().put(settings).put(SETTING_CREATION_DATE, creationDate).build(); return this; diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/SearchSlowLog.java b/core/src/main/java/org/elasticsearch/index/search/stats/SearchSlowLog.java index f782399a6e2..72aaa33c3d7 100644 --- a/core/src/main/java/org/elasticsearch/index/search/stats/SearchSlowLog.java +++ b/core/src/main/java/org/elasticsearch/index/search/stats/SearchSlowLog.java @@ -190,13 +190,7 @@ public final class SearchSlowLog{ } sb.append("search_type[").append(context.searchType()).append("], total_shards[").append(context.numberOfShards()).append("], "); if (context.request().source() != null) { - // try { - // // - // sb.append("source[").append(XContentHelper.convertToJson(context.request().source(), - // // reformat)).append("], "); NOCOMMIT fix this - // } catch (IOException e) { - // sb.append("source[_failed_to_convert_], "); - // } + sb.append("source[").append(context.request().source()).append("], "); } else { sb.append("source[], "); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java index 79efe13ed2f..198c3204ee4 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/warmer/put/RestPutWarmerAction.java @@ -19,14 +19,27 @@ package org.elasticsearch.rest.action.admin.indices.warmer.put; import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.support.AcknowledgedRestListener; +import org.elasticsearch.rest.action.support.RestActions; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.io.IOException; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; @@ -35,9 +48,12 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT; */ public class RestPutWarmerAction extends BaseRestHandler { + private final IndicesQueriesRegistry queryRegistry; + @Inject - public RestPutWarmerAction(Settings settings, RestController controller, Client client) { + public RestPutWarmerAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry) { super(settings, controller, client); + this.queryRegistry = queryRegistry; controller.registerHandler(PUT, "/_warmer/{name}", this); controller.registerHandler(PUT, "/{index}/_warmer/{name}", this); controller.registerHandler(PUT, "/{index}/{type}/_warmer/{name}", this); @@ -56,16 +72,19 @@ public class RestPutWarmerAction extends BaseRestHandler { } @Override - public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException { PutWarmerRequest putWarmerRequest = new PutWarmerRequest(request.param("name")); - // SearchRequest searchRequest = new - // SearchRequest(Strings.splitStringByCommaToArray(request.param("index"))) - // .types(Strings.splitStringByCommaToArray(request.param("type"))) - // .requestCache(request.paramAsBoolean("request_cache", null)) - // .source(request.content()); - // searchRequest.indicesOptions(IndicesOptions.fromRequest(request, - // searchRequest.indicesOptions())); - // putWarmerRequest.searchRequest(searchRequest); NOCOMMIT fix this + + BytesReference sourceBytes = RestActions.getRestContent(request); + XContentParser parser = XContentFactory.xContent(sourceBytes).createParser(sourceBytes); + QueryParseContext queryParseContext = new QueryParseContext(new Index(""), queryRegistry); // NORELEASE remove index + queryParseContext.reset(parser); + SearchSourceBuilder source = SearchSourceBuilder.PROTOTYPE.fromXContent(parser, queryParseContext); + SearchRequest searchRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index"))) + .types(Strings.splitStringByCommaToArray(request.param("type"))) + .requestCache(request.paramAsBoolean("request_cache", null)).source(source); + searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); + putWarmerRequest.searchRequest(searchRequest); putWarmerRequest.timeout(request.paramAsTime("timeout", putWarmerRequest.timeout())); putWarmerRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putWarmerRequest.masterNodeTimeout())); client.admin().indices().putWarmer(putWarmerRequest, new AcknowledgedRestListener<>(channel)); diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 3d767bc29b1..3c90d54f5f3 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -26,9 +26,15 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.Index; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; @@ -43,6 +49,8 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.suggest.SuggestBuilder; +import java.io.IOException; + import static org.elasticsearch.common.unit.TimeValue.parseTimeValue; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -53,9 +61,12 @@ import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion; */ public class RestSearchAction extends BaseRestHandler { + private final IndicesQueriesRegistry queryRegistry; + @Inject - public RestSearchAction(Settings settings, RestController controller, Client client) { + public RestSearchAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry) { super(settings, controller, client); + this.queryRegistry = queryRegistry; controller.registerHandler(GET, "/_search", this); controller.registerHandler(POST, "/_search", this); controller.registerHandler(GET, "/{index}/_search", this); @@ -79,13 +90,13 @@ public class RestSearchAction extends BaseRestHandler { } @Override - public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException { SearchRequest searchRequest; - searchRequest = RestSearchAction.parseSearchRequest(request, parseFieldMatcher); + searchRequest = RestSearchAction.parseSearchRequest(request, parseFieldMatcher, queryRegistry); client.search(searchRequest, new RestStatusToXContentListener(channel)); } - public static SearchRequest parseSearchRequest(RestRequest request, ParseFieldMatcher parseFieldMatcher) { + public static SearchRequest parseSearchRequest(RestRequest request, ParseFieldMatcher parseFieldMatcher, IndicesQueriesRegistry queryRegistry) throws IOException { String[] indices = Strings.splitStringByCommaToArray(request.param("index")); SearchRequest searchRequest = new SearchRequest(indices); // get the content, and put it in the body @@ -95,8 +106,11 @@ public class RestSearchAction extends BaseRestHandler { if (isTemplateRequest) { searchRequest.templateSource(RestActions.getRestContent(request)); } else { - // searchRequest.source(RestActions.getRestContent(request)); - // NOCOMMIT fix this + BytesReference sourceBytes = RestActions.getRestContent(request); + XContentParser parser = XContentFactory.xContent(sourceBytes).createParser(sourceBytes); + QueryParseContext queryParseContext = new QueryParseContext(new Index(""), queryRegistry); // NORELEASE remove index + queryParseContext.reset(parser); + searchRequest.source(SearchSourceBuilder.PROTOTYPE.fromXContent(parser, queryParseContext)); } } diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index 53a56d235b7..d7530e6f0aa 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -30,6 +30,7 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.search.TopDocs; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cache.recycler.PageCacheRecycler; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -47,6 +48,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.ConcurrentMapLong; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; @@ -76,6 +78,7 @@ import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script.ScriptParseException; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.SearchScript; import org.elasticsearch.script.Template; import org.elasticsearch.script.mustache.MustacheScriptEngineService; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -86,11 +89,16 @@ import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.QueryFetchSearchResult; import org.elasticsearch.search.fetch.ScrollQueryFetchSearchResult; import org.elasticsearch.search.fetch.ShardFetchRequest; +import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext; +import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext.FieldDataField; +import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; +import org.elasticsearch.search.fetch.script.ScriptFieldsContext.ScriptField; import org.elasticsearch.search.internal.DefaultSearchContext; import org.elasticsearch.search.internal.InternalScrollSearchRequest; import org.elasticsearch.search.internal.ScrollContext; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.SearchContext.Lifetime; +import org.elasticsearch.search.internal.ShardSearchLocalRequest; import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.query.QueryPhase; import org.elasticsearch.search.query.QuerySearchRequest; @@ -101,6 +109,7 @@ import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -748,54 +757,65 @@ public class SearchService extends AbstractLifecycleComponent { context.aggregations(null); // NOCOMMIT parse source.aggregations() // ByteReference into // SearchContextAggregations object - context.suggest(null); // NOCOMMIT parse source.suggest() ByteReference - // into SuggestionSearchContext object + XContentParser suggestParser = null; + try { + suggestParser = XContentFactory.xContent(source.suggest()).createParser(source.suggest()); + this.elementParsers.get("suggest").parse(suggestParser, context); + } catch (Exception e) { + String sSource = "_na_"; + try { + sSource = source.toString(); + } catch (Throwable e1) { + // ignore + } + XContentLocation location = suggestParser != null ? suggestParser.getTokenLocation() : null; + throw new SearchParseException(context, "failed to parse suggest source [" + sSource + "]", location, e); + } context.addRescore(null);// NOCOMMIT parse source.rescore() // ByteReference into RescoreSearchContext // object - // NOCOMMIT populate the rest of the search request + context.fieldNames().addAll(source.fields()); context.explain(source.explain()); + context.fetchSourceContext(source.fetchSource()); + FieldDataFieldsContext fieldDataFieldsContext = context.getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY); + for (String field : source.fieldDataFields()) { + fieldDataFieldsContext.add(new FieldDataField(field)); + } + XContentParser highlighterParser = null; + try { + highlighterParser = XContentFactory.xContent(source.highlighter()).createParser(source.highlighter()); + this.elementParsers.get("highlight").parse(highlighterParser, context); + } catch (Exception e) { + String sSource = "_na_"; + try { + sSource = source.toString(); + } catch (Throwable e1) { + // ignore + } + XContentLocation location = highlighterParser != null ? highlighterParser.getTokenLocation() : null; + throw new SearchParseException(context, "failed to parse suggest source [" + sSource + "]", location, e); + } + XContentParser innerHitsParser = null; + try { + innerHitsParser = XContentFactory.xContent(source.innerHits()).createParser(source.innerHits()); + this.elementParsers.get("highlight").parse(innerHitsParser, context); + } catch (Exception e) { + String sSource = "_na_"; + try { + sSource = source.toString(); + } catch (Throwable e1) { + // ignore + } + XContentLocation location = innerHitsParser != null ? innerHitsParser.getTokenLocation() : null; + throw new SearchParseException(context, "failed to parse suggest source [" + sSource + "]", location, e); + } + for (org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField field : source.scriptFields()) { + SearchScript searchScript = context.scriptService().search(context.lookup(), field.script(), ScriptContext.Standard.SEARCH); + context.scriptFields().add(new ScriptField(field.fieldName(), searchScript, false)); // NORELEASE need to have ignore_exception parsed somewhere + } + // NOCOMMIT need to work out what to do about term_vectors_fetch (previously handled by TermVectorsFetchParseElement) as this is not available as an option in SearchSourceBuilder context.version(source.version()); - -// XContentParser parser = null; -// try { -// parser = XContentFactory.xContent(source).createParser(source); -// XContentParser.Token token; -// token = parser.nextToken(); -// if (token != XContentParser.Token.START_OBJECT) { -// throw new ElasticsearchParseException("failed to parse search source. source must be an object, but found [{}] instead", token.name()); -// } -// while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { -// if (token == XContentParser.Token.FIELD_NAME) { -// String fieldName = parser.currentName(); -// parser.nextToken(); -// SearchParseElement element = elementParsers.get(fieldName); -// if (element == null) { -// throw new SearchParseException(context, "failed to parse search source. unknown search element [" + fieldName + "]", parser.getTokenLocation()); -// } -// element.parse(parser, context); -// } else { -// if (token == null) { -// throw new ElasticsearchParseException("failed to parse search source. end of query source reached but query is not complete."); -// } else { -// throw new ElasticsearchParseException("failed to parse search source. expected field name but got [{}]", token); -// } -// } -// } -// } catch (Throwable e) { -// String sSource = "_na_"; -// try { -// sSource = XContentHelper.convertToJson(source, false); -// } catch (Throwable e1) { -// // ignore -// } -// XContentLocation location = parser != null ? parser.getTokenLocation() : null; -// throw new SearchParseException(context, "failed to parse search source [" + sSource + "]", location, e); -// } finally { -// if (parser != null) { -// parser.close(); -// } -// } + context.groupStats(Arrays.asList(source.stats())); // NORELEASE stats should be a list in SearchSourceBuilder } private static final int[] EMPTY_DOC_IDS = new int[0]; @@ -1088,32 +1108,26 @@ public class SearchService extends AbstractLifecycleComponent { SearchContext context = null; try { long now = System.nanoTime(); - // ShardSearchRequest request = new - // ShardSearchLocalRequest(indexShard.shardId(), - // indexMetaData.numberOfShards(), - // SearchType.QUERY_THEN_FETCH, entry.source(), - // entry.types(), entry.requestCache()); - // context = createContext(request, - // warmerContext.searcher()); - // // if we use sort, we need to do query to sort on + ShardSearchRequest request = new ShardSearchLocalRequest(indexShard.shardId(), indexMetaData.numberOfShards(), + SearchType.QUERY_THEN_FETCH, entry.source(), entry.types(), entry.requestCache()); + context = createContext(request, warmerContext.searcher()); + // if we use sort, we need to do query to sort on // it and load relevant field data - // // if not, we might as well set size=0 (and cache + // if not, we might as well set size=0 (and cache // if needed) - // if (context.sort() == null) { - // context.size(0); - // } - // boolean canCache = - // indicesQueryCache.canCache(request, context); - // // early terminate when we can cache, since we + if (context.sort() == null) { + context.size(0); + } + boolean canCache = indicesQueryCache.canCache(request, context); + // early terminate when we can cache, since we // can only do proper caching on top level searcher - // // also, if we can't cache, and its top, we don't + // also, if we can't cache, and its top, we don't // need to execute it, since we already did when its // not top - // if (canCache != top) { - // return; - // } - // loadOrExecuteQueryPhase(request, context, - // queryPhase); NOCOMMIT fix this + if (canCache != top) { + return; + } + loadOrExecuteQueryPhase(request, context, queryPhase); long took = System.nanoTime() - now; if (indexShard.warmerService().logger().isTraceEnabled()) { indexShard.warmerService().logger().trace("warmed [{}], took [{}]", entry.name(), TimeValue.timeValueNanos(took)); diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index bc611782137..9b01564dd27 100644 --- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -1022,7 +1022,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } } - private static class ScriptField implements Writeable, ToXContent { + public static class ScriptField implements Writeable, ToXContent { public static final ScriptField PROTOTYPE = new ScriptField(null, null); diff --git a/core/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java b/core/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java index 6e881cb1473..4ff4ed1ae9f 100644 --- a/core/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java +++ b/core/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java @@ -23,8 +23,6 @@ import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; @@ -32,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.ArrayList; @@ -67,10 +66,10 @@ public class IndexWarmersMetaData extends AbstractDiffable public static class Entry { private final String name; private final String[] types; - private final BytesReference source; + private final SearchSourceBuilder source; private final Boolean requestCache; - public Entry(String name, String[] types, Boolean requestCache, BytesReference source) { + public Entry(String name, String[] types, Boolean requestCache, SearchSourceBuilder source) { this.name = name; this.types = types == null ? Strings.EMPTY_ARRAY : types; this.source = source; @@ -86,7 +85,7 @@ public class IndexWarmersMetaData extends AbstractDiffable } @Nullable - public BytesReference source() { + public SearchSourceBuilder source() { return this.source; } @@ -141,9 +140,9 @@ public class IndexWarmersMetaData extends AbstractDiffable for (int i = 0; i < entries.length; i++) { String name = in.readString(); String[] types = in.readStringArray(); - BytesReference source = null; + SearchSourceBuilder source = null; if (in.readBoolean()) { - source = in.readBytesReference(); + source = SearchSourceBuilder.PROTOTYPE.readFrom(in); } Boolean queryCache; queryCache = in.readOptionalBoolean(); @@ -162,7 +161,7 @@ public class IndexWarmersMetaData extends AbstractDiffable out.writeBoolean(false); } else { out.writeBoolean(true); - out.writeBytesReference(entry.source()); + entry.source.writeTo(out); } out.writeOptionalBoolean(entry.requestCache()); } @@ -194,7 +193,7 @@ public class IndexWarmersMetaData extends AbstractDiffable } else if (token == XContentParser.Token.START_OBJECT) { String name = currentFieldName; List types = new ArrayList<>(2); - BytesReference source = null; + SearchSourceBuilder source = null; Boolean queryCache = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -207,13 +206,12 @@ public class IndexWarmersMetaData extends AbstractDiffable } } else if (token == XContentParser.Token.START_OBJECT) { if ("source".equals(currentFieldName)) { - XContentBuilder builder = XContentFactory.jsonBuilder().map(parser.mapOrdered()); - source = builder.bytes(); - } - } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { - if ("source".equals(currentFieldName)) { - source = new BytesArray(parser.binaryValue()); + source = SearchSourceBuilder.PROTOTYPE.fromXContent(parser, null); // NOCOMMIT need context from somewhere } +// } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { +// if ("source".equals(currentFieldName)) { +// source = new BytesArray(parser.binaryValue()); +// } NORELEASE do we need this? } else if (token.isValue()) { if ("requestCache".equals(currentFieldName) || "request_cache".equals(currentFieldName)) { queryCache = parser.booleanValue(); @@ -239,22 +237,12 @@ public class IndexWarmersMetaData extends AbstractDiffable } public static void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { - boolean binary = params.paramAsBoolean("binary", false); builder.startObject(entry.name(), XContentBuilder.FieldCaseConversion.NONE); builder.field("types", entry.types()); if (entry.requestCache() != null) { builder.field("requestCache", entry.requestCache()); } - builder.field("source"); - if (binary) { - builder.value(entry.source()); - } else { - Map mapping; - try (XContentParser parser = XContentFactory.xContent(entry.source()).createParser(entry.source())) { - mapping = parser.mapOrdered(); - } - builder.map(mapping); - } + builder.field("source", entry.source()); builder.endObject(); } diff --git a/core/src/test/java/org/elasticsearch/action/count/CountRequestTests.java b/core/src/test/java/org/elasticsearch/action/count/CountRequestTests.java index 483a4f0ac67..baca298d302 100644 --- a/core/src/test/java/org/elasticsearch/action/count/CountRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/count/CountRequestTests.java @@ -31,6 +31,7 @@ import org.junit.Test; import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; public class CountRequestTests extends ESTestCase { @@ -75,11 +76,7 @@ public class CountRequestTests extends ESTestCase { if (countRequest.source() == null) { assertThat(searchRequest.source(), nullValue()); } else { - // Map sourceMap = - // XContentHelper.convertToMap(searchRequest.source(), false).v2(); - // assertThat(sourceMap.size(), equalTo(1)); - // assertThat(sourceMap.get("query"), notNullValue()); NOCOMMIT fix - // this + assertThat(searchRequest.source().query(), notNullValue()); } Map extraSourceMap = XContentHelper.convertToMap(searchRequest.extraSource(), false).v2(); diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index 1aa1602b7e5..66001f66e45 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.cluster; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.google.common.collect.ImmutableMap; + import org.elasticsearch.Version; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -39,6 +40,7 @@ import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.test.ESIntegTestCase; import org.junit.Test; @@ -531,7 +533,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase { randomName("warm"), new String[]{randomName("type")}, randomBoolean(), - new BytesArray(randomAsciiOfLength(1000))) + new SearchSourceBuilder()) // NOCOMMIT this used to be new BytesArray(randomAsciiOfLength(1000)) whiat should it be now? ); } else { return new IndexWarmersMetaData(); diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java index 9cf2034e05a..492370d6ef3 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java @@ -48,10 +48,10 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.suggest.SuggestRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.suggest.SuggestBuilders; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.test.ESIntegTestCase; @@ -61,7 +61,9 @@ import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuild import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class IndicesOptionsIntegrationIT extends ESIntegTestCase { @@ -508,7 +510,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { .setIndicesOptions(IndicesOptions.lenientExpandOpen()) .execute().actionGet(); assertHitCount(response, 0l); - + //you should still be able to run empty searches without things blowing up response = client().prepareSearch() .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -613,7 +615,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareExists("bar").get().isExists(), equalTo(false)); assertThat(client().admin().indices().prepareExists("barbaz").get().isExists(), equalTo(false)); } - + @Test public void testPutWarmer() throws Exception { createIndex("foobar"); @@ -622,26 +624,26 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareGetWarmers("foobar").setWarmers("warmer1").get().getWarmers().size(), equalTo(1)); } - + @Test public void testPutWarmer_wildcard() throws Exception { createIndex("foo", "foobar", "bar", "barbaz"); ensureYellow(); verify(client().admin().indices().preparePutWarmer("warmer1").setSearchRequest(client().prepareSearch().setIndices("foo*").setQuery(QueryBuilders.matchAllQuery())), false); - + assertThat(client().admin().indices().prepareGetWarmers("foo").setWarmers("warmer1").get().getWarmers().size(), equalTo(1)); assertThat(client().admin().indices().prepareGetWarmers("foobar").setWarmers("warmer1").get().getWarmers().size(), equalTo(1)); assertThat(client().admin().indices().prepareGetWarmers("bar").setWarmers("warmer1").get().getWarmers().size(), equalTo(0)); assertThat(client().admin().indices().prepareGetWarmers("barbaz").setWarmers("warmer1").get().getWarmers().size(), equalTo(0)); verify(client().admin().indices().preparePutWarmer("warmer2").setSearchRequest(client().prepareSearch().setIndices().setQuery(QueryBuilders.matchAllQuery())), false); - + assertThat(client().admin().indices().prepareGetWarmers("foo").setWarmers("warmer2").get().getWarmers().size(), equalTo(1)); assertThat(client().admin().indices().prepareGetWarmers("foobar").setWarmers("warmer2").get().getWarmers().size(), equalTo(1)); assertThat(client().admin().indices().prepareGetWarmers("bar").setWarmers("warmer2").get().getWarmers().size(), equalTo(1)); assertThat(client().admin().indices().prepareGetWarmers("barbaz").setWarmers("warmer2").get().getWarmers().size(), equalTo(1)); - + } @Test @@ -652,7 +654,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("foobar").get().exists(), equalTo(true)); } - + @Test public void testPutAlias_wildcard() throws Exception { createIndex("foo", "foobar", "bar", "barbaz"); @@ -669,14 +671,13 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("foobar").get().exists(), equalTo(true)); assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("bar").get().exists(), equalTo(true)); assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("barbaz").get().exists(), equalTo(true)); - + } - + @Test public void testDeleteWarmer() throws Exception { - IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry( - "test1", new String[]{"typ1"}, false, new BytesArray("{\"query\" : { \"match_all\" : {}}}") - ); + SearchSourceBuilder source = new SearchSourceBuilder(); + IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "typ1" }, false, source); assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry))); ensureYellow(); @@ -690,9 +691,8 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { public void testDeleteWarmer_wildcard() throws Exception { verify(client().admin().indices().prepareDeleteWarmer().setIndices("_all").setNames("test1"), true); - IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry( - "test1", new String[]{"type1"}, false, new BytesArray("{\"query\" : { \"match_all\" : {}}}") - ); + SearchSourceBuilder source = new SearchSourceBuilder(); + IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "type1" }, false, source); assertAcked(prepareCreate("foo").addCustom(new IndexWarmersMetaData(entry))); assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry))); assertAcked(prepareCreate("bar").addCustom(new IndexWarmersMetaData(entry))); @@ -737,7 +737,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { assertThat(client().admin().indices().prepareGetMappings("foobar").get().mappings().get("foobar").get("type3"), notNullValue()); assertThat(client().admin().indices().prepareGetMappings("bar").get().mappings().get("bar").get("type3"), notNullValue()); assertThat(client().admin().indices().prepareGetMappings("barbaz").get().mappings().get("barbaz").get("type3"), notNullValue()); - + verify(client().admin().indices().preparePutMapping("c*").setType("type1").setSource("field", "type=string"), true); @@ -883,7 +883,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { private static void verify(ActionRequestBuilder requestBuilder, boolean fail) { verify(requestBuilder, fail, 0); } - + private static void verify(ActionRequestBuilder requestBuilder, boolean fail, long expectedCount) { if (fail) { if (requestBuilder instanceof MultiSearchRequestBuilder) {