diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy index 7b949b3e1da..54699a52e76 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy @@ -39,9 +39,6 @@ class PluginPropertiesExtension { @Input String classname - @Input - boolean isolated = true - PluginPropertiesExtension(Project project) { name = project.name version = project.version diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy index b5128817fb0..856c9255312 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy @@ -54,12 +54,6 @@ class PluginPropertiesTask extends Copy { if (extension.classname == null) { throw new InvalidUserDataException('classname is a required setting for esplugin') } - doFirst { - if (extension.isolated == false) { - String warning = "WARNING: Disabling plugin isolation in ${project.path} is deprecated and will be removed in the future" - logger.warn("${'=' * warning.length()}\n${warning}\n${'=' * warning.length()}") - } - } // configure property substitution from(templateFile) into(generatedResourcesDir) @@ -80,7 +74,6 @@ class PluginPropertiesTask extends Copy { 'version': stringSnap(extension.version), 'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch), 'javaVersion': project.targetCompatibility as String, - 'isolated': extension.isolated as String, 'classname': extension.classname ] } diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 13b983c8a5e..af83e2db0e8 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -660,7 +660,6 @@ - diff --git a/buildSrc/src/main/resources/plugin-descriptor.properties b/buildSrc/src/main/resources/plugin-descriptor.properties index 7659f11bd09..ebde46d326b 100644 --- a/buildSrc/src/main/resources/plugin-descriptor.properties +++ b/buildSrc/src/main/resources/plugin-descriptor.properties @@ -38,12 +38,3 @@ java.version=${javaVersion} # # 'elasticsearch.version' version of elasticsearch compiled against elasticsearch.version=${elasticsearchVersion} -# -### deprecated elements for jvm plugins : -# -# 'isolated': true if the plugin should have its own classloader. -# passing false is deprecated, and only intended to support plugins -# that have hard dependencies against each other. If this is -# not specified, then the plugin is isolated by default. -isolated=${isolated} -# \ No newline at end of file diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 39c32192052..13c0885a61a 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,4 +1,4 @@ -elasticsearch = 5.0.0 +elasticsearch = 5.0.0-alpha1 lucene = 6.0.0-snapshot-f0aa4fc # optional dependencies diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index 8b65adf170d..fe794d96661 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -64,9 +64,9 @@ public class Version { public static final Version V_2_2_1 = new Version(V_2_2_1_ID, org.apache.lucene.util.Version.LUCENE_5_4_1); public static final int V_2_3_0_ID = 2030099; public static final Version V_2_3_0 = new Version(V_2_3_0_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); - public static final int V_5_0_0_ID = 5000099; - public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_0_0); - public static final Version CURRENT = V_5_0_0; + public static final int V_5_0_0_alpha1_ID = 5000001; + public static final Version V_5_0_0_alpha1 = new Version(V_5_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_6_0_0); + public static final Version CURRENT = V_5_0_0_alpha1; static { assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to [" @@ -79,8 +79,8 @@ public class Version { public static Version fromId(int id) { switch (id) { - case V_5_0_0_ID: - return V_5_0_0; + case V_5_0_0_alpha1_ID: + return V_5_0_0_alpha1; case V_2_3_0_ID: return V_2_3_0; case V_2_2_1_ID: diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java index be9387f9a8a..2b33a669428 100644 --- a/core/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java @@ -173,8 +173,6 @@ import org.elasticsearch.action.search.TransportClearScrollAction; import org.elasticsearch.action.search.TransportMultiSearchAction; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchScrollAction; -import org.elasticsearch.action.suggest.SuggestAction; -import org.elasticsearch.action.suggest.TransportSuggestAction; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; @@ -320,7 +318,6 @@ public class ActionModule extends AbstractModule { registerAction(MultiTermVectorsAction.INSTANCE, TransportMultiTermVectorsAction.class, TransportShardMultiTermsVectorAction.class); registerAction(DeleteAction.INSTANCE, TransportDeleteAction.class); - registerAction(SuggestAction.INSTANCE, TransportSuggestAction.class); registerAction(UpdateAction.INSTANCE, TransportUpdateAction.class); registerAction(MultiGetAction.INSTANCE, TransportMultiGetAction.class, TransportShardMultiGetAction.class); diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java index 336f4c84596..9dbe4ee1aeb 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java @@ -84,7 +84,7 @@ public class TransportCancelTasksAction extends TransportTasksAction operation) { - if (request.getTaskId().isSet() == false) { + if (request.getTaskId().isSet()) { // we are only checking one task, we can optimize it CancellableTask task = taskManager.getCancellableTask(request.getTaskId().getId()); if (task != null) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java index c027bfa7ab4..ad9fa9509e8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java @@ -178,7 +178,7 @@ public class TaskInfo implements Writeable, ToXContent { } builder.dateValueField("start_time_in_millis", "start_time", startTime); builder.timeValueField("running_time_in_nanos", "running_time", runningTimeNanos, TimeUnit.NANOSECONDS); - if (parentTaskId.isSet() == false) { + if (parentTaskId.isSet()) { builder.field("parent_task_id", parentTaskId.toString()); } return builder; diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java index d5175e263e7..e6ea002a794 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java @@ -84,7 +84,13 @@ public class TransportListTasksAction extends TransportTasksAction { while (System.nanoTime() - timeoutTime < 0) { - if (taskManager.getTask(t.getId()) == null) { + Task task = taskManager.getTask(t.getId()); + if (task == null) { + return; + } + if (task.getAction().startsWith(ListTasksAction.NAME)) { + // It doesn't make sense to wait for List Tasks and it can cause an infinite loop of the task waiting + // for itself of one of its child tasks return; } try { diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index b6ee76b16ed..b130e6b378f 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -42,7 +42,6 @@ import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.index.suggest.stats.SuggestStats; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.index.warmer.WarmerStats; import org.elasticsearch.indices.IndicesQueryCache; @@ -109,7 +108,7 @@ public class CommonStats implements Streamable, ToXContent { translog = new TranslogStats(); break; case Suggest: - suggest = new SuggestStats(); + // skip break; case RequestCache: requestCache = new RequestCacheStats(); @@ -177,7 +176,7 @@ public class CommonStats implements Streamable, ToXContent { translog = indexShard.translogStats(); break; case Suggest: - suggest = indexShard.suggestStats(); + // skip break; case RequestCache: requestCache = indexShard.requestCache().stats(); @@ -236,9 +235,6 @@ public class CommonStats implements Streamable, ToXContent { @Nullable public TranslogStats translog; - @Nullable - public SuggestStats suggest; - @Nullable public RequestCacheStats requestCache; @@ -367,14 +363,6 @@ public class CommonStats implements Streamable, ToXContent { } else { translog.add(stats.getTranslog()); } - if (suggest == null) { - if (stats.getSuggest() != null) { - suggest = new SuggestStats(); - suggest.add(stats.getSuggest()); - } - } else { - suggest.add(stats.getSuggest()); - } if (requestCache == null) { if (stats.getRequestCache() != null) { requestCache = new RequestCacheStats(); @@ -468,11 +456,6 @@ public class CommonStats implements Streamable, ToXContent { return translog; } - @Nullable - public SuggestStats getSuggest() { - return suggest; - } - @Nullable public RequestCacheStats getRequestCache() { return requestCache; @@ -555,7 +538,6 @@ public class CommonStats implements Streamable, ToXContent { segments = SegmentsStats.readSegmentsStats(in); } translog = in.readOptionalStreamable(TranslogStats::new); - suggest = in.readOptionalStreamable(SuggestStats::new); requestCache = in.readOptionalStreamable(RequestCacheStats::new); recoveryStats = in.readOptionalStreamable(RecoveryStats::new); } @@ -647,7 +629,6 @@ public class CommonStats implements Streamable, ToXContent { segments.writeTo(out); } out.writeOptionalStreamable(translog); - out.writeOptionalStreamable(suggest); out.writeOptionalStreamable(requestCache); out.writeOptionalStreamable(recoveryStats); } @@ -700,9 +681,6 @@ public class CommonStats implements Streamable, ToXContent { if (translog != null) { translog.toXContent(builder, params); } - if (suggest != null) { - suggest.toXContent(builder, params); - } if (requestCache != null) { requestCache.toXContent(builder, params); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java index c67c35a4108..e2f250dd577 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java @@ -190,7 +190,7 @@ public class CommonStatsFlags implements Streamable, Cloneable { out.writeStringArrayNullable(groups); out.writeStringArrayNullable(fieldDataFields); out.writeStringArrayNullable(completionDataFields); - if (out.getVersion().onOrAfter(Version.V_5_0_0)) { + if (out.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) { out.writeBoolean(includeSegmentFileSizes); } } @@ -208,7 +208,7 @@ public class CommonStatsFlags implements Streamable, Cloneable { groups = in.readStringArray(); fieldDataFields = in.readStringArray(); completionDataFields = in.readStringArray(); - if (in.getVersion().onOrAfter(Version.V_5_0_0)) { + if (in.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) { includeSegmentFileSizes = in.readBoolean(); } else { includeSegmentFileSizes = false; @@ -244,7 +244,7 @@ public class CommonStatsFlags implements Streamable, Cloneable { Completion("completion"), Segments("segments"), Translog("translog"), - Suggest("suggest"), + Suggest("suggest"), // unused RequestCache("request_cache"), Recovery("recovery"); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java index 0ae21a3ac09..cad919cbd18 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java @@ -152,11 +152,6 @@ public class IndicesStatsRequestBuilder extends BroadcastOperationRequestBuilder return this; } - public IndicesStatsRequestBuilder setSuggest(boolean suggest) { - request.suggest(suggest); - return this; - } - public IndicesStatsRequestBuilder setRequestCache(boolean requestCache) { request.requestCache(requestCache); return this; diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java index 83757dab089..9286601da69 100644 --- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java @@ -21,7 +21,6 @@ package org.elasticsearch.action.percolate; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; @@ -127,7 +126,7 @@ public class PercolateRequestBuilder extends ActionRequestBuilder sort) { sourceBuilder().addSort(sort); return this; } diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java index d687d48fb0c..5a5924f7883 100644 --- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java @@ -48,13 +48,13 @@ import java.util.Map; public class PercolateSourceBuilder extends ToXContentToBytes { private DocBuilder docBuilder; - private QueryBuilder queryBuilder; + private QueryBuilder queryBuilder; private Integer size; - private List sorts; + private List> sorts; private Boolean trackScores; private HighlightBuilder highlightBuilder; private List> aggregationBuilders; - private List pipelineAggregationBuilders; + private List> pipelineAggregationBuilders; /** * Sets the document to run the percolate queries against. @@ -68,7 +68,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes { * Sets a query to reduce the number of percolate queries to be evaluated and score the queries that match based * on this query. */ - public PercolateSourceBuilder setQueryBuilder(QueryBuilder queryBuilder) { + public PercolateSourceBuilder setQueryBuilder(QueryBuilder queryBuilder) { this.queryBuilder = queryBuilder; return this; } @@ -98,7 +98,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes { * * By default the matching percolator queries are returned in an undefined order. */ - public PercolateSourceBuilder addSort(SortBuilder sort) { + public PercolateSourceBuilder addSort(SortBuilder sort) { if (sorts == null) { sorts = new ArrayList<>(); } @@ -137,7 +137,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes { /** * Add an aggregation definition. */ - public PercolateSourceBuilder addAggregation(PipelineAggregatorBuilder aggregationBuilder) { + public PercolateSourceBuilder addAggregation(PipelineAggregatorBuilder aggregationBuilder) { if (pipelineAggregationBuilders == null) { pipelineAggregationBuilders = new ArrayList<>(); } @@ -160,10 +160,8 @@ public class PercolateSourceBuilder extends ToXContentToBytes { } if (sorts != null) { builder.startArray("sort"); - for (SortBuilder sort : sorts) { - builder.startObject(); + for (SortBuilder sort : sorts) { sort.toXContent(builder, params); - builder.endObject(); } builder.endArray(); } @@ -182,7 +180,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes { } } if (pipelineAggregationBuilders != null) { - for (PipelineAggregatorBuilder aggregation : pipelineAggregationBuilders) { + for (PipelineAggregatorBuilder aggregation : pipelineAggregationBuilders) { aggregation.toXContent(builder, params); } } diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 9d3c200ed98..e87d393bf05 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -295,6 +295,13 @@ public class SearchRequest extends ActionRequest implements Indic return this.requestCache; } + /** + * @return true if the request only has suggest + */ + public boolean isSuggestOnly() { + return source != null && source.isSuggestOnly(); + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 8fe99c1e590..5b20a05bad5 100644 --- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Set; import static org.elasticsearch.action.search.SearchType.QUERY_AND_FETCH; +import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH; /** * @@ -72,6 +73,17 @@ public class TransportSearchAction extends HandledTransportAction { - - public static final SuggestAction INSTANCE = new SuggestAction(); - public static final String NAME = "indices:data/read/suggest"; - - private SuggestAction() { - super(NAME); - } - - @Override - public SuggestResponse newResponse() { - return new SuggestResponse(new Suggest()); - } - - @Override - public SuggestRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new SuggestRequestBuilder(client, this); - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequest.java b/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequest.java deleted file mode 100644 index 1398dd1dcf1..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequest.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.suggest; - -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.support.broadcast.BroadcastRequest; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.search.suggest.SuggestBuilder; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Objects; - -/** - * A request to get suggestions for corrections of phrases. Best created with - * {@link org.elasticsearch.client.Requests#suggestRequest(String...)}. - *

- * The request requires the suggest query source to be set using - * {@link #suggest(org.elasticsearch.search.suggest.SuggestBuilder)} - * - * @see SuggestResponse - * @see org.elasticsearch.client.Client#suggest(SuggestRequest) - * @see org.elasticsearch.client.Requests#suggestRequest(String...) - * @see org.elasticsearch.search.suggest.SuggestBuilders - */ -public final class SuggestRequest extends BroadcastRequest { - - @Nullable - private String routing; - - @Nullable - private String preference; - - private SuggestBuilder suggest; - - public SuggestRequest() { - } - - /** - * Constructs a new suggest request against the provided indices. No indices provided means it will - * run against all indices. - */ - public SuggestRequest(String... indices) { - super(indices); - } - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = super.validate(); - return validationException; - } - - /** - * The suggestion query to get correction suggestions for - */ - public SuggestBuilder suggest() { - return suggest; - } - - /** - * set a new source for the suggest query - */ - public SuggestRequest suggest(SuggestBuilder suggest) { - Objects.requireNonNull(suggest, "suggest must not be null"); - this.suggest = suggest; - return this; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public String routing() { - return this.routing; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public SuggestRequest routing(String routing) { - this.routing = routing; - return this; - } - - /** - * The routing values to control the shards that the search will be executed on. - */ - public SuggestRequest routing(String... routings) { - this.routing = Strings.arrayToCommaDelimitedString(routings); - return this; - } - - public SuggestRequest preference(String preference) { - this.preference = preference; - return this; - } - - public String preference() { - return this.preference; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - routing = in.readOptionalString(); - preference = in.readOptionalString(); - suggest = SuggestBuilder.PROTOTYPE.readFrom(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - Objects.requireNonNull(suggest, "suggest must not be null"); - super.writeTo(out); - out.writeOptionalString(routing); - out.writeOptionalString(preference); - suggest.writeTo(out); - } - - @Override - public String toString() { - Objects.requireNonNull(suggest, "suggest must not be null"); - String sSource = "_na_"; - try { - XContentBuilder builder = JsonXContent.contentBuilder(); - builder = suggest.toXContent(builder, ToXContent.EMPTY_PARAMS); - sSource = builder.string(); - } catch (Exception e) { - // ignore - } - return "[" + Arrays.toString(indices) + "]" + ", suggest[" + sSource + "]"; - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequestBuilder.java deleted file mode 100644 index b64745bf400..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/SuggestRequestBuilder.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.suggest; - -import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.search.suggest.SuggestBuilder; -import org.elasticsearch.search.suggest.SuggestionBuilder; - -/** - * A suggest action request builder. - */ -public class SuggestRequestBuilder extends BroadcastOperationRequestBuilder { - - final SuggestBuilder suggest = new SuggestBuilder(); - - public SuggestRequestBuilder(ElasticsearchClient client, SuggestAction action) { - super(client, action, new SuggestRequest()); - } - - /** - * Add a definition for suggestions to the request - * @param name the name for the suggestion that will also be used in the response - * @param suggestion the suggestion configuration - */ - public SuggestRequestBuilder addSuggestion(String name, SuggestionBuilder suggestion) { - suggest.addSuggestion(name, suggestion); - return this; - } - - /** - * A comma separated list of routing values to control the shards the search will be executed on. - */ - public SuggestRequestBuilder setRouting(String routing) { - request.routing(routing); - return this; - } - - public SuggestRequestBuilder setSuggestText(String globalText) { - this.suggest.setGlobalText(globalText); - return this; - } - - /** - * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, - * _shards:x,y to operate on shards x & y, or a custom value, which guarantees that the same order - * will be used across different requests. - */ - public SuggestRequestBuilder setPreference(String preference) { - request.preference(preference); - return this; - } - - /** - * The routing values to control the shards that the search will be executed on. - */ - public SuggestRequestBuilder setRouting(String... routing) { - request.routing(routing); - return this; - } - - @Override - protected SuggestRequest beforeExecute(SuggestRequest request) { - request.suggest(suggest); - return request; - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/SuggestResponse.java b/core/src/main/java/org/elasticsearch/action/suggest/SuggestResponse.java deleted file mode 100644 index 445e804b5b5..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/SuggestResponse.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.suggest; - -import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.BroadcastResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.search.suggest.Suggest; - -import java.io.IOException; -import java.util.List; - -import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; - -/** - * The response of the suggest action. - */ -public final class SuggestResponse extends BroadcastResponse { - - private final Suggest suggest; - - SuggestResponse(Suggest suggest) { - this.suggest = suggest; - } - - SuggestResponse(Suggest suggest, int totalShards, int successfulShards, int failedShards, List shardFailures) { - super(totalShards, successfulShards, failedShards, shardFailures); - this.suggest = suggest; - } - - /** - * The Suggestions of the phrase. - */ - public Suggest getSuggest() { - return suggest; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - this.suggest.readFrom(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - this.suggest.writeTo(out); - } - - @Override - public String toString() { - try { - XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); - builder.startObject(); - suggest.toXContent(builder, EMPTY_PARAMS); - builder.endObject(); - return builder.string(); - } catch (IOException e) { - return "{ \"error\" : \"" + e.getMessage() + "\"}"; - } - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java b/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java deleted file mode 100644 index 95bf111ed71..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.suggest; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ShardOperationFailedException; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.DefaultShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.TransportBroadcastAction; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.GroupShardsIterator; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.suggest.stats.ShardSuggestMetric; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.search.suggest.Suggest; -import org.elasticsearch.search.suggest.SuggestBuilder; -import org.elasticsearch.search.suggest.SuggestPhase; -import org.elasticsearch.search.suggest.SuggestionSearchContext; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReferenceArray; - -/** - * Defines the transport of a suggestion request across the cluster - */ -public class TransportSuggestAction - extends TransportBroadcastAction { - - private final IndicesService indicesService; - private final SuggestPhase suggestPhase; - - @Inject - public TransportSuggestAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, - TransportService transportService, IndicesService indicesService, SuggestPhase suggestPhase, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { - super(settings, SuggestAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, - SuggestRequest::new, ShardSuggestRequest::new, ThreadPool.Names.SUGGEST); - this.indicesService = indicesService; - this.suggestPhase = suggestPhase; - } - - @Override - protected ShardSuggestRequest newShardRequest(int numShards, ShardRouting shard, SuggestRequest request) { - return new ShardSuggestRequest(shard.shardId(), request); - } - - @Override - protected ShardSuggestResponse newShardResponse() { - return new ShardSuggestResponse(); - } - - @Override - protected GroupShardsIterator shards(ClusterState clusterState, SuggestRequest request, String[] concreteIndices) { - Map> routingMap = - indexNameExpressionResolver.resolveSearchRouting(clusterState, request.routing(), request.indices()); - return clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference()); - } - - @Override - protected ClusterBlockException checkGlobalBlock(ClusterState state, SuggestRequest request) { - return state.blocks().globalBlockedException(ClusterBlockLevel.READ); - } - - @Override - protected ClusterBlockException checkRequestBlock(ClusterState state, SuggestRequest countRequest, String[] concreteIndices) { - return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices); - } - - @Override - protected SuggestResponse newResponse(SuggestRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) { - int successfulShards = 0; - int failedShards = 0; - - final Map> groupedSuggestions = new HashMap<>(); - - List shardFailures = null; - for (int i = 0; i < shardsResponses.length(); i++) { - Object shardResponse = shardsResponses.get(i); - if (shardResponse == null) { - // simply ignore non active shards - } else if (shardResponse instanceof BroadcastShardOperationFailedException) { - failedShards++; - if (shardFailures == null) { - shardFailures = new ArrayList<>(); - } - shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse)); - } else { - Suggest suggest = ((ShardSuggestResponse) shardResponse).getSuggest(); - Suggest.group(groupedSuggestions, suggest); - successfulShards++; - } - } - - return new SuggestResponse(new Suggest(Suggest.reduce(groupedSuggestions)), shardsResponses.length(), - successfulShards, failedShards, shardFailures); - } - - @Override - protected ShardSuggestResponse shardOperation(ShardSuggestRequest request) { - IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); - IndexShard indexShard = indexService.getShard(request.shardId().id()); - ShardSuggestMetric suggestMetric = indexShard.getSuggestMetric(); - suggestMetric.preSuggest(); - long startTime = System.nanoTime(); - try (Engine.Searcher searcher = indexShard.acquireSearcher("suggest")) { - SuggestBuilder suggest = request.suggest(); - if (suggest != null) { - final SuggestionSearchContext context = suggest.build(indexService.newQueryShardContext()); - final Suggest result = suggestPhase.execute(context, searcher.searcher()); - return new ShardSuggestResponse(request.shardId(), result); - } - return new ShardSuggestResponse(request.shardId(), new Suggest()); - } catch (Throwable ex) { - throw new ElasticsearchException("failed to execute suggest", ex); - } finally { - suggestMetric.postSuggest(System.nanoTime() - startTime); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/action/suggest/package-info.java b/core/src/main/java/org/elasticsearch/action/suggest/package-info.java deleted file mode 100644 index a2c0f48ea51..00000000000 --- a/core/src/main/java/org/elasticsearch/action/suggest/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * Suggest action. - */ -package org.elasticsearch.action.suggest; \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java index f1045387259..dc296a84720 100644 --- a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java +++ b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java @@ -60,7 +60,7 @@ public class BaseTasksRequest> extends @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - if (taskId.isSet() == false && nodesIds.length > 0) { + if (taskId.isSet() && nodesIds.length > 0) { validationException = addValidationError("task id cannot be used together with node ids", validationException); } @@ -165,12 +165,12 @@ public class BaseTasksRequest> extends if (getActions() != null && getActions().length > 0 && Regex.simpleMatch(getActions(), task.getAction()) == false) { return false; } - if (getTaskId().isSet() == false) { + if (getTaskId().isSet()) { if(getTaskId().getId() != task.getId()) { return false; } } - if (parentTaskId.isSet() == false) { + if (parentTaskId.isSet()) { if (parentTaskId.equals(task.getParentTaskId()) == false) { return false; } diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index a14c6e00e14..97678e6c060 100644 --- a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -125,14 +125,14 @@ public abstract class TransportTasksAction< protected String[] resolveNodes(TasksRequest request, ClusterState clusterState) { if (request.getTaskId().isSet()) { - return clusterState.nodes().resolveNodesIds(request.getNodesIds()); - } else { return new String[]{request.getTaskId().getNodeId()}; + } else { + return clusterState.nodes().resolveNodesIds(request.getNodesIds()); } } protected void processTasks(TasksRequest request, Consumer operation) { - if (request.getTaskId().isSet() == false) { + if (request.getTaskId().isSet()) { // we are only checking one task, we can optimize it Task task = taskManager.getTask(request.getTaskId().getId()); if (task != null) { diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java index 5f4e980d68e..576ce720f35 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java @@ -123,7 +123,9 @@ final class BootstrapCheck { if (Constants.LINUX) { checks.add(new MaxNumberOfThreadsCheck()); } - checks.add(new MaxSizeVirtualMemoryCheck()); + if (Constants.LINUX || Constants.MAC_OS_X) { + checks.add(new MaxSizeVirtualMemoryCheck()); + } return Collections.unmodifiableList(checks); } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 0cc952907c0..bb1f6cc87d5 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -76,6 +76,9 @@ class Elasticsearch extends Command { @Override protected void execute(Terminal terminal, OptionSet options) throws Exception { + if (options.nonOptionArguments().isEmpty() == false) { + throw new UserError(ExitCodes.USAGE, "Positional arguments not allowed, found " + options.nonOptionArguments()); + } if (options.has(versionOption)) { if (options.has(daemonizeOption) || options.has(pidfileOption)) { throw new UserError(ExitCodes.USAGE, "Elasticsearch version option is mutually exclusive with any other option"); diff --git a/core/src/main/java/org/elasticsearch/client/Client.java b/core/src/main/java/org/elasticsearch/client/Client.java index e5d8d4f55b7..1c7676bc91c 100644 --- a/core/src/main/java/org/elasticsearch/client/Client.java +++ b/core/src/main/java/org/elasticsearch/client/Client.java @@ -68,9 +68,6 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.action.suggest.SuggestRequest; -import org.elasticsearch.action.suggest.SuggestRequestBuilder; -import org.elasticsearch.action.suggest.SuggestResponse; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.MultiTermVectorsRequestBuilder; import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; @@ -367,29 +364,6 @@ public interface Client extends ElasticsearchClient, Releasable { */ MultiGetRequestBuilder prepareMultiGet(); - /** - * Suggestion matching a specific phrase. - * - * @param request The suggest request - * @return The result future - * @see Requests#suggestRequest(String...) - */ - ActionFuture suggest(SuggestRequest request); - - /** - * Suggestions matching a specific phrase. - * - * @param request The suggest request - * @param listener A listener to be notified of the result - * @see Requests#suggestRequest(String...) - */ - void suggest(SuggestRequest request, ActionListener listener); - - /** - * Suggestions matching a specific phrase. - */ - SuggestRequestBuilder prepareSuggest(String... indices); - /** * Search across one or more indices and one or more types with a query. * diff --git a/core/src/main/java/org/elasticsearch/client/Requests.java b/core/src/main/java/org/elasticsearch/client/Requests.java index 53c6abd971c..276bd9d9062 100644 --- a/core/src/main/java/org/elasticsearch/client/Requests.java +++ b/core/src/main/java/org/elasticsearch/client/Requests.java @@ -60,9 +60,7 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.action.suggest.SuggestRequest; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.search.suggest.SuggestBuilder; /** * A handy one stop shop for creating requests (make sure to import static this class). @@ -126,16 +124,6 @@ public class Requests { return new GetRequest(index); } - /** - * Creates a suggest request for getting suggestions from provided indices. - * The suggest query has to be set using {@link org.elasticsearch.action.suggest.SuggestRequest#suggest(SuggestBuilder)}. - * @param indices The indices to suggest from. Use null or _all to execute against all indices - * @see org.elasticsearch.client.Client#suggest(org.elasticsearch.action.suggest.SuggestRequest) - */ - public static SuggestRequest suggestRequest(String... indices) { - return new SuggestRequest(indices); - } - /** * Creates a search request against one or more indices. Note, the search source must be set either using the * actual JSON search source, or the {@link org.elasticsearch.search.builder.SearchSourceBuilder}. diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java index f729d5287df..0044890ee35 100644 --- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -314,10 +314,6 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollAction; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.SearchScrollRequestBuilder; -import org.elasticsearch.action.suggest.SuggestAction; -import org.elasticsearch.action.suggest.SuggestRequest; -import org.elasticsearch.action.suggest.SuggestRequestBuilder; -import org.elasticsearch.action.suggest.SuggestResponse; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.action.termvectors.MultiTermVectorsAction; @@ -660,21 +656,6 @@ public abstract class AbstractClient extends AbstractComponent implements Client return new MultiSearchRequestBuilder(this, MultiSearchAction.INSTANCE); } - @Override - public ActionFuture suggest(final SuggestRequest request) { - return execute(SuggestAction.INSTANCE, request); - } - - @Override - public void suggest(final SuggestRequest request, final ActionListener listener) { - execute(SuggestAction.INSTANCE, request, listener); - } - - @Override - public SuggestRequestBuilder prepareSuggest(String... indices) { - return new SuggestRequestBuilder(this, SuggestAction.INSTANCE).setIndices(indices); - } - @Override public ActionFuture termVectors(final TermVectorsRequest request) { return execute(TermVectorsAction.INSTANCE, request); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index a5975deb9cf..269a238e710 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -266,7 +266,7 @@ public final class ShardRouting implements Streamable, ToXContent { return false; } - if (indexMetaData.activeAllocationIds(id()).isEmpty() && indexMetaData.getCreationVersion().onOrAfter(Version.V_5_0_0)) { + if (indexMetaData.activeAllocationIds(id()).isEmpty() && indexMetaData.getCreationVersion().onOrAfter(Version.V_5_0_0_alpha1)) { // when no shards with this id have ever been active for this index return false; } diff --git a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java index a41b0388af7..16ce91dc38f 100644 --- a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java +++ b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java @@ -445,5 +445,10 @@ public class PagedBytesReference implements BytesReference { // do nothing } + @Override + public int available() throws IOException { + return length - pos; + } + } } diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java index 5f3bd011dd9..b8132b4e870 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java @@ -59,6 +59,11 @@ public abstract class FilterStreamInput extends StreamInput { delegate.close(); } + @Override + public int available() throws IOException { + return delegate.available(); + } + @Override public Version getVersion() { return delegate.getVersion(); diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java index e9aa52cf4d0..d786041af49 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java @@ -74,6 +74,11 @@ public class InputStreamStreamInput extends StreamInput { is.close(); } + @Override + public int available() throws IOException { + return is.available(); + } + @Override public int read() throws IOException { return is.read(); diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index aca136a2a9a..c5709db5363 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -37,14 +37,13 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; -import org.elasticsearch.ingest.IngestStats; -import org.elasticsearch.search.rescore.RescoreBuilder; -import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.completion.context.QueryContext; -import org.elasticsearch.search.suggest.phrase.SmoothingModel; -import org.elasticsearch.tasks.Task; import org.elasticsearch.search.aggregations.AggregatorBuilder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder; +import org.elasticsearch.search.rescore.RescoreBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.phrase.SmoothingModel; +import org.elasticsearch.tasks.Task; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -68,7 +67,6 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.ElasticsearchException.readException; @@ -375,6 +373,9 @@ public abstract class StreamInput extends InputStream { @Override public abstract void close() throws IOException; + @Override + public abstract int available() throws IOException; + public String[] readStringArray() throws IOException { int size = readVInt(); if (size == 0) { @@ -687,21 +688,21 @@ public abstract class StreamInput extends InputStream { /** * Reads a {@link AggregatorBuilder} from the current stream */ - public AggregatorBuilder readAggregatorFactory() throws IOException { + public AggregatorBuilder readAggregatorFactory() throws IOException { return readNamedWriteable(AggregatorBuilder.class); } /** * Reads a {@link PipelineAggregatorBuilder} from the current stream */ - public PipelineAggregatorBuilder readPipelineAggregatorFactory() throws IOException { + public PipelineAggregatorBuilder readPipelineAggregatorFactory() throws IOException { return readNamedWriteable(PipelineAggregatorBuilder.class); } /** * Reads a {@link QueryBuilder} from the current stream */ - public QueryBuilder readQuery() throws IOException { + public QueryBuilder readQuery() throws IOException { return readNamedWriteable(QueryBuilder.class); } @@ -726,6 +727,13 @@ public abstract class StreamInput extends InputStream { return readNamedWriteable(SuggestionBuilder.class); } + /** + * Reads a {@link SortBuilder} from the current stream + */ + public SortBuilder readSortBuilder() throws IOException { + return readNamedWriteable(SortBuilder.class); + } + /** * Reads a {@link org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder} from the current stream */ diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index dd357e27709..630c297edf6 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -36,13 +36,13 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; -import org.elasticsearch.search.rescore.RescoreBuilder; -import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.completion.context.QueryContext; -import org.elasticsearch.search.suggest.phrase.SmoothingModel; -import org.elasticsearch.tasks.Task; import org.elasticsearch.search.aggregations.AggregatorBuilder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder; +import org.elasticsearch.search.rescore.RescoreBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.phrase.SmoothingModel; +import org.elasticsearch.tasks.Task; import org.joda.time.ReadableInstant; import java.io.EOFException; @@ -532,7 +532,7 @@ public abstract class StreamOutput extends OutputStream { } } - public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOException { + public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOException { if (writeable != null) { writeBoolean(true); writeable.writeTo(this); @@ -663,7 +663,7 @@ public abstract class StreamOutput extends OutputStream { /** * Writes a {@link NamedWriteable} to the current stream, by first writing its name and then the object itself */ - void writeNamedWriteable(NamedWriteable namedWriteable) throws IOException { + void writeNamedWriteable(NamedWriteable namedWriteable) throws IOException { writeString(namedWriteable.getWriteableName()); namedWriteable.writeTo(this); } @@ -685,7 +685,7 @@ public abstract class StreamOutput extends OutputStream { /** * Writes a {@link QueryBuilder} to the current stream */ - public void writeQuery(QueryBuilder queryBuilder) throws IOException { + public void writeQuery(QueryBuilder queryBuilder) throws IOException { writeNamedWriteable(queryBuilder); } @@ -745,8 +745,15 @@ public abstract class StreamOutput extends OutputStream { /** * Writes a {@link SuggestionBuilder} to the current stream */ - public void writeSuggestion(SuggestionBuilder suggestion) throws IOException { + public void writeSuggestion(SuggestionBuilder suggestion) throws IOException { writeNamedWriteable(suggestion); } + /** + * Writes a {@link SortBuilder} to the current stream + */ + public void writeSortBuilder(SortBuilder sort) throws IOException { + writeNamedWriteable(sort); + } + } diff --git a/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java b/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java index a210f3ae6d5..70e6807cb92 100644 --- a/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java +++ b/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java @@ -76,8 +76,9 @@ public class PropertyPlaceholder { * @param placeholderResolver the PlaceholderResolver to use for replacement. * @return the supplied value with placeholders replaced inline. */ - public String replacePlaceholders(String value, PlaceholderResolver placeholderResolver) { - Objects.requireNonNull(value, "Argument 'value' must not be null."); + public String replacePlaceholders(String key, String value, PlaceholderResolver placeholderResolver) { + Objects.requireNonNull(key); + Objects.requireNonNull(value, "value can not be null for [" + key + "]"); return parseStringValue(value, placeholderResolver, new HashSet()); } diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java index f0e1b2e64ea..a2ebe7a2c30 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -523,6 +523,28 @@ public class Setting extends ToXContentToBytes { return new Setting<>(key, defaultValue, (s) -> ByteSizeValue.parseBytesSizeValue(s, key), properties); } + public static Setting byteSizeSetting(String key, ByteSizeValue value, ByteSizeValue minValue, ByteSizeValue maxValue, + Property... properties) { + return byteSizeSetting(key, (s) -> value.toString(), minValue, maxValue, properties); + } + + public static Setting byteSizeSetting(String key, Function defaultValue, + ByteSizeValue minValue, ByteSizeValue maxValue, + Property... properties) { + return new Setting<>(key, defaultValue, (s) -> parseByteSize(s, minValue, maxValue, key), properties); + } + + public static ByteSizeValue parseByteSize(String s, ByteSizeValue minValue, ByteSizeValue maxValue, String key) { + ByteSizeValue value = ByteSizeValue.parseBytesSizeValue(s, key); + if (value.bytes() < minValue.bytes()) { + throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue); + } + if (value.bytes() > maxValue.bytes()) { + throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be =< " + maxValue); + } + return value; + } + public static Setting positiveTimeSetting(String key, TimeValue defaultValue, Property... properties) { return timeSetting(key, defaultValue, TimeValue.timeValueMillis(0), properties); } diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java index a6784e561d2..ce79bf92d20 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -1221,7 +1221,7 @@ public final class Settings implements ToXContent { } }; for (Map.Entry entry : new HashMap<>(map).entrySet()) { - String value = propertyPlaceholder.replacePlaceholders(entry.getValue(), placeholderResolver); + String value = propertyPlaceholder.replacePlaceholders(entry.getKey(), entry.getValue(), placeholderResolver); // if the values exists and has length, we should maintain it in the map // otherwise, the replace process resolved into removing it if (Strings.hasLength(value)) { diff --git a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java index a456da0779d..e7447e301c4 100644 --- a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java +++ b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java @@ -118,7 +118,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent { final boolean enoughAllocationsFound; if (lastActiveAllocationIds.isEmpty()) { - assert Version.indexCreated(indexMetaData.getSettings()).before(Version.V_5_0_0) : "trying to allocated a primary with an empty allocation id set, but index is new"; + assert Version.indexCreated(indexMetaData.getSettings()).before(Version.V_5_0_0_alpha1) : "trying to allocated a primary with an empty allocation id set, but index is new"; // when we load an old index (after upgrading cluster) or restore a snapshot of an old index // fall back to old version-based allocation mode // Note that once the shard has been active, lastActiveAllocationIds will be non-empty @@ -128,7 +128,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent { } else { enoughAllocationsFound = isEnoughVersionBasedAllocationsFound(indexMetaData, nodeShardsResult); } - logger.debug("[{}][{}]: version-based allocation for pre-{} index found {} allocations of {}", shard.index(), shard.id(), Version.V_5_0_0, nodeShardsResult.allocationsFound, shard); + logger.debug("[{}][{}]: version-based allocation for pre-{} index found {} allocations of {}", shard.index(), shard.id(), Version.V_5_0_0_alpha1, nodeShardsResult.allocationsFound, shard); } else { assert lastActiveAllocationIds.isEmpty() == false; // use allocation ids to select nodes diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java index 453552b9dd1..b9146df8c96 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java @@ -127,7 +127,7 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable } if (analyzers.containsKey("default_index")) { final Version createdVersion = indexSettings.getIndexVersionCreated(); - if (createdVersion.onOrAfter(Version.V_5_0_0)) { + if (createdVersion.onOrAfter(Version.V_5_0_0_alpha1)) { throw new IllegalArgumentException("setting [index.analysis.analyzer.default_index] is not supported anymore, use [index.analysis.analyzer.default] instead for index [" + index().getName() + "]"); } else { deprecationLogger.deprecated("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] instead for index [{}]", index().getName()); diff --git a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java index d2c4a3c14c0..10948f35d54 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java +++ b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java @@ -348,7 +348,7 @@ public class SegmentsStats implements Streamable, ToXContent { indexWriterMaxMemoryInBytes = in.readLong(); bitsetMemoryInBytes = in.readLong(); - if (in.getVersion().onOrAfter(Version.V_5_0_0)) { + if (in.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) { int size = in.readVInt(); ImmutableOpenMap.Builder map = ImmutableOpenMap.builder(size); for (int i = 0; i < size; i++) { @@ -376,7 +376,7 @@ public class SegmentsStats implements Streamable, ToXContent { out.writeLong(indexWriterMaxMemoryInBytes); out.writeLong(bitsetMemoryInBytes); - if (out.getVersion().onOrAfter(Version.V_5_0_0)) { + if (out.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) { out.writeVInt(fileSizes.size()); for (Iterator> it = fileSizes.iterator(); it.hasNext();) { ObjectObjectCursor entry = it.next(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 9f6d0ea270a..8077a732142 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -219,7 +219,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable { } protected boolean defaultDocValues(Version indexCreated) { - if (indexCreated.onOrAfter(Version.V_5_0_0)) { + if (indexCreated.onOrAfter(Version.V_5_0_0_alpha1)) { // add doc values by default to keyword (boolean, numerics, etc.) fields return fieldType.tokenized() == false; } else { @@ -229,7 +229,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable { protected void setupFieldType(BuilderContext context) { fieldType.setName(buildFullName(context)); - if (context.indexCreatedVersion().before(Version.V_5_0_0)) { + if (context.indexCreatedVersion().before(Version.V_5_0_0_alpha1)) { fieldType.setOmitNorms(fieldType.omitNorms() && fieldType.boost() == 1.0f); } if (fieldType.indexAnalyzer() == null && fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE) { @@ -289,7 +289,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable { if (!customBoost() // don't set boosts eg. on dv fields && field.fieldType().indexOptions() != IndexOptions.NONE - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(fieldType().boost()); } context.doc().add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 054f307d3dd..9eafa8ccd71 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -341,7 +341,7 @@ public abstract class MappedFieldType extends FieldType { public Query termQuery(Object value, @Nullable QueryShardContext context) { TermQuery query = new TermQuery(createTerm(value)); if (boost == 1f || - (context != null && context.indexVersionCreated().before(Version.V_5_0_0))) { + (context != null && context.indexVersionCreated().before(Version.V_5_0_0_alpha1))) { return query; } return new BoostQuery(query, boost); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java index 29c37eee271..02d6a536812 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java @@ -110,8 +110,6 @@ public class BooleanFieldMapper extends FieldMapper { } builder.nullValue(lenientNodeBooleanValue(propNode)); iterator.remove(); - } else if (parseMultiField(builder, name, parserContext, propName, propNode)) { - iterator.remove(); } } return builder; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java index 2c7555c7fe6..c42f0b7cd42 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java @@ -261,7 +261,7 @@ public class ByteFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Byte objValue = fieldType().nullValue(); @@ -294,7 +294,7 @@ public class ByteFieldMapper extends NumberFieldMapper { } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomByteNumericField field = new CustomByteNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 7c6381b7c1c..5d66fb62c76 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -488,7 +488,7 @@ public class DateFieldMapper extends NumberFieldMapper { } else if (token == XContentParser.Token.VALUE_NUMBER) { dateAsString = parser.text(); } else if (token == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { String currentFieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -523,7 +523,7 @@ public class DateFieldMapper extends NumberFieldMapper { if (value != null) { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomLongNumericField field = new CustomLongNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index 02bdfdf1dd1..69d56439b33 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -253,7 +253,7 @@ public class DoubleFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Double objValue = fieldType().nullValue(); @@ -287,7 +287,7 @@ public class DoubleFieldMapper extends NumberFieldMapper { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomDoubleNumericField field = new CustomDoubleNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index 4182b37b08f..48b9ca95875 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -265,7 +265,7 @@ public class FloatFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Float objValue = fieldType().nullValue(); @@ -299,7 +299,7 @@ public class FloatFieldMapper extends NumberFieldMapper { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomFloatNumericField field = new CustomFloatNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java index 3e1e7a538a2..860377d0879 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java @@ -270,7 +270,7 @@ public class IntegerFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Integer objValue = fieldType().nullValue(); @@ -307,7 +307,7 @@ public class IntegerFieldMapper extends NumberFieldMapper { protected void addIntegerFields(ParseContext context, List fields, int value, float boost) { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomIntegerNumericField field = new CustomIntegerNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java index a3da79341a0..ab36b738c83 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java @@ -134,8 +134,6 @@ public final class KeywordFieldMapper extends FieldMapper implements AllFieldMap } else if (propName.equals("eager_global_ordinals")) { builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode)); iterator.remove(); - } else if (parseMultiField(builder, name, parserContext, propName, propNode)) { - iterator.remove(); } } return builder; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java index 8c346c2e0c1..b261df7a221 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java @@ -258,7 +258,7 @@ public class LongFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Long objValue = fieldType().nullValue(); @@ -291,7 +291,7 @@ public class LongFieldMapper extends NumberFieldMapper { } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomLongNumericField field = new CustomLongNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index 2f5bd3d974c..eb00e0744c8 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -266,7 +266,7 @@ public class ShortFieldMapper extends NumberFieldMapper { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; Short objValue = fieldType().nullValue(); @@ -299,7 +299,7 @@ public class ShortFieldMapper extends NumberFieldMapper { } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomShortNumericField field = new CustomShortNumericField(value, fieldType()); - if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (boost != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(boost); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java index f58db38a3dd..c87efcd9218 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java @@ -193,7 +193,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc @Override public Mapper.Builder parse(String fieldName, Map node, ParserContext parserContext) throws MapperParsingException { - if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0)) { + if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha1)) { // Automatically upgrade simple mappings for ease of upgrade, otherwise fail if (SUPPORTED_PARAMETERS_FOR_AUTO_UPGRADE.containsAll(node.keySet())) { deprecationLogger.deprecated("The [string] field is deprecated, please use [text] or [keyword] instead on [{}]", @@ -334,8 +334,6 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize); DocumentMapperParser.checkNoRemainingFields(propName, frequencyFilter, parserContext.indexVersionCreated()); iterator.remove(); - } else if (parseMultiField(builder, fieldName, parserContext, propName, propNode)) { - iterator.remove(); } } return builder; @@ -490,7 +488,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc int positionIncrementGap, int ignoreAbove, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); - if (Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0)) { + if (Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha1)) { throw new IllegalArgumentException("The [string] type is removed in 5.0. You should now use either a [text] " + "or [keyword] field instead for field [" + fieldType.name() + "]"); } @@ -573,7 +571,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { Field field = new Field(fieldType().name(), valueAndBoost.value(), fieldType()); - if (valueAndBoost.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (valueAndBoost.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(valueAndBoost.boost()); } fields.add(field); @@ -600,7 +598,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc return new ValueAndBoost(nullValue, defaultBoost); } if (parser.currentToken() == XContentParser.Token.START_OBJECT - && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { XContentParser.Token token; String currentFieldName = null; String value = nullValue; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java index 582ae6aff38..0baa4564b69 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java @@ -163,8 +163,6 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize); DocumentMapperParser.checkNoRemainingFields(propName, frequencyFilter, parserContext.indexVersionCreated()); iterator.remove(); - } else if (parseMultiField(builder, fieldName, parserContext, propName, propNode)) { - iterator.remove(); } } return builder; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java index 47471254e13..bc7c97bc4ff 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java @@ -250,7 +250,7 @@ public class TypeParsers { } else if (propName.equals("boost")) { builder.boost(nodeFloatValue(propNode)); iterator.remove(); - } else if (parserContext.indexVersionCreated().before(Version.V_5_0_0) + } else if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1) && parseNorms(builder, propName, propNode, parserContext)) { iterator.remove(); } else if (propName.equals("index_options")) { @@ -265,9 +265,11 @@ public class TypeParsers { iterator.remove(); } else if (propName.equals("fielddata") && propNode instanceof Map - && parserContext.indexVersionCreated().before(Version.V_5_0_0)) { + && parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)) { // ignore for bw compat iterator.remove(); + } else if (parseMultiField(builder, name, parserContext, propName, propNode)) { + iterator.remove(); } else if (propName.equals("copy_to")) { if (parserContext.isWithinMultiField()) { if (indexVersionCreated.after(Version.V_2_1_0) || @@ -434,7 +436,7 @@ public class TypeParsers { } private static SimilarityProvider resolveSimilarity(Mapper.TypeParser.ParserContext parserContext, String name, String value) { - if (parserContext.indexVersionCreated().before(Version.V_5_0_0) && "default".equals(value)) { + if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1) && "default".equals(value)) { // "default" similarity has been renamed into "classic" in 3.x. value = SimilarityService.DEFAULT_SIMILARITY; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java index ae181b3c98c..fec0720ce70 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java @@ -220,8 +220,6 @@ public abstract class BaseGeoPointFieldMapper extends FieldMapper implements Arr } else if (propName.equals(Names.IGNORE_MALFORMED)) { builder.ignoreMalformed(XContentMapValues.lenientNodeBooleanValue(propNode)); iterator.remove(); - } else if (parseMultiField(builder, name, parserContext, propName, propNode)) { - iterator.remove(); } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java index 57778fa8d25..1a0f2561a40 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java @@ -46,7 +46,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.core.DoubleFieldMapper; import java.io.IOException; import java.util.Iterator; @@ -454,7 +453,7 @@ public class GeoShapeFieldMapper extends FieldMapper { } for (Field field : fields) { if (!customBoost() && - fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(fieldType().boost()); } context.doc().add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java index a102f03a5f1..3eeca0352a7 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java @@ -124,7 +124,7 @@ public class SourceFieldMapper extends MetadataFieldMapper { if (fieldName.equals("enabled")) { builder.enabled(lenientNodeBooleanValue(fieldNode)); iterator.remove(); - } else if ("format".equals(fieldName) && parserContext.indexVersionCreated().before(Version.V_5_0_0)) { + } else if ("format".equals(fieldName) && parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)) { // ignore on old indices, reject on and after 5.0 iterator.remove(); } else if (fieldName.equals("includes")) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index d0ba6aaf3ab..b754c163669 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -322,7 +322,7 @@ public class IpFieldMapper extends NumberFieldMapper { final long value = ipToLong(ipAsString); if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomLongNumericField field = new CustomLongNumericField(value, fieldType()); - if (fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0)) { + if (fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { field.setBoost(fieldType().boost()); } fields.add(field); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java b/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java index 4a1ab2027b7..3deb487e50b 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/object/DynamicTemplate.java @@ -41,12 +41,20 @@ public class DynamicTemplate implements ToXContent { public static enum MatchType { SIMPLE { + @Override + public boolean matches(String pattern, String value) { + return Regex.simpleMatch(pattern, value); + } @Override public String toString() { return "simple"; } }, REGEX { + @Override + public boolean matches(String pattern, String value) { + return value.matches(pattern); + } @Override public String toString() { return "regex"; @@ -61,6 +69,9 @@ public class DynamicTemplate implements ToXContent { } throw new IllegalArgumentException("No matching pattern matched on [" + value + "]"); } + + /** Whether {@code value} matches {@code regex}. */ + public abstract boolean matches(String regex, String value); } public static DynamicTemplate parse(String name, Map conf, @@ -89,7 +100,7 @@ public class DynamicTemplate implements ToXContent { matchPattern = entry.getValue().toString(); } else if ("mapping".equals(propName)) { mapping = (Map) entry.getValue(); - } else if (indexVersionCreated.onOrAfter(Version.V_5_0_0)) { + } else if (indexVersionCreated.onOrAfter(Version.V_5_0_0_alpha1)) { // unknown parameters were ignored before but still carried through serialization // so we need to ignore them at parsing time for old indices throw new IllegalArgumentException("Illegal dynamic template parameter: [" + propName + "]"); @@ -137,23 +148,23 @@ public class DynamicTemplate implements ToXContent { } public boolean match(ContentPath path, String name, String dynamicType) { - if (pathMatch != null && !patternMatch(pathMatch, path.pathAsText(name))) { + if (pathMatch != null && !matchType.matches(pathMatch, path.pathAsText(name))) { return false; } - if (match != null && !patternMatch(match, name)) { + if (match != null && !matchType.matches(match, name)) { return false; } - if (pathUnmatch != null && patternMatch(pathUnmatch, path.pathAsText(name))) { + if (pathUnmatch != null && matchType.matches(pathUnmatch, path.pathAsText(name))) { return false; } - if (unmatch != null && patternMatch(unmatch, name)) { + if (unmatch != null && matchType.matches(unmatch, name)) { return false; } if (matchMappingType != null) { if (dynamicType == null) { return false; } - if (!patternMatch(matchMappingType, dynamicType)) { + if (!matchType.matches(matchMappingType, dynamicType)) { return false; } } @@ -186,13 +197,6 @@ public class DynamicTemplate implements ToXContent { return type; } - private boolean patternMatch(String pattern, String str) { - if (matchType == MatchType.SIMPLE) { - return Regex.simpleMatch(pattern, str); - } - return str.matches(pattern); - } - public Map mappingForName(String name, String dynamicType) { return processMap(mapping, name, dynamicType); } diff --git a/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java b/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java index 8ee9d55c312..a8669c98cd8 100644 --- a/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java +++ b/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.PrefixCodedTerms; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -92,10 +93,17 @@ public final class ExtractQueryTermsService { * an UnsupportedQueryException is thrown. */ static Set extractQueryTerms(Query query) { - // TODO: add support for the TermsQuery when it has methods to access the actual terms it encapsulates // TODO: add support for span queries if (query instanceof TermQuery) { return Collections.singleton(((TermQuery) query).getTerm()); + } else if (query instanceof TermsQuery) { + Set terms = new HashSet<>(); + TermsQuery termsQuery = (TermsQuery) query; + PrefixCodedTerms.TermIterator iterator = termsQuery.getTermData().iterator(); + for (BytesRef term = iterator.next(); term != null; term = iterator.next()) { + terms.add(new Term(iterator.field(), term)); + } + return terms; } else if (query instanceof PhraseQuery) { Term[] terms = ((PhraseQuery) query).getTerms(); if (terms.length == 0) { diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java index 459ab1df0b8..78d8af06827 100644 --- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java +++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java @@ -29,7 +29,6 @@ import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Query; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; @@ -37,11 +36,9 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; -import org.elasticsearch.common.lucene.index.ElasticsearchLeafReader; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -143,7 +140,7 @@ public final class PercolatorQueryCache extends AbstractIndexComponent } IntObjectHashMap queries = new IntObjectHashMap<>(); - boolean legacyLoading = indexVersionCreated.before(Version.V_5_0_0); + boolean legacyLoading = indexVersionCreated.before(Version.V_5_0_0_alpha1); PostingsEnum postings = leafReader.postings(new Term(TypeFieldMapper.NAME, PercolatorFieldMapper.TYPE_NAME), PostingsEnum.NONE); if (postings != null) { if (legacyLoading) { diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java index 5cb1e54d203..7ca70eef6aa 100644 --- a/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java @@ -301,7 +301,7 @@ public class PercolatorQueryBuilder extends AbstractQueryBuilder { + if (searchContext.hasOnlySuggest()) { + statsHolder.suggestCurrent.inc(); + } else { + statsHolder.queryCurrent.inc(); } - } + }); } public void onFailedQueryPhase(SearchContext searchContext) { - totalStats.queryCurrent.dec(); - if (searchContext.groupStats() != null) { - for (int i = 0; i < searchContext.groupStats().size(); i++) { - groupStats(searchContext.groupStats().get(i)).queryCurrent.dec(); + computeStats(searchContext, statsHolder -> { + if (searchContext.hasOnlySuggest()) { + statsHolder.suggestCurrent.dec(); + } else { + statsHolder.queryCurrent.dec(); } - } + }); } public void onQueryPhase(SearchContext searchContext, long tookInNanos) { - totalStats.queryMetric.inc(tookInNanos); - totalStats.queryCurrent.dec(); - if (searchContext.groupStats() != null) { - for (int i = 0; i < searchContext.groupStats().size(); i++) { - StatsHolder statsHolder = groupStats(searchContext.groupStats().get(i)); + computeStats(searchContext, statsHolder -> { + if (searchContext.hasOnlySuggest()) { + statsHolder.suggestMetric.inc(tookInNanos); + statsHolder.suggestCurrent.dec(); + } else { statsHolder.queryMetric.inc(tookInNanos); statsHolder.queryCurrent.dec(); } - } + }); slowLogSearchService.onQueryPhase(searchContext, tookInNanos); } public void onPreFetchPhase(SearchContext searchContext) { - totalStats.fetchCurrent.inc(); - if (searchContext.groupStats() != null) { - for (int i = 0; i < searchContext.groupStats().size(); i++) { - groupStats(searchContext.groupStats().get(i)).fetchCurrent.inc(); - } - } + computeStats(searchContext, statsHolder -> statsHolder.fetchCurrent.inc()); } public void onFailedFetchPhase(SearchContext searchContext) { - totalStats.fetchCurrent.dec(); - if (searchContext.groupStats() != null) { - for (int i = 0; i < searchContext.groupStats().size(); i++) { - groupStats(searchContext.groupStats().get(i)).fetchCurrent.dec(); - } - } + computeStats(searchContext, statsHolder -> statsHolder.fetchCurrent.dec()); } public void onFetchPhase(SearchContext searchContext, long tookInNanos) { - totalStats.fetchMetric.inc(tookInNanos); - totalStats.fetchCurrent.dec(); - if (searchContext.groupStats() != null) { - for (int i = 0; i < searchContext.groupStats().size(); i++) { - StatsHolder statsHolder = groupStats(searchContext.groupStats().get(i)); - statsHolder.fetchMetric.inc(tookInNanos); - statsHolder.fetchCurrent.dec(); - } - } + computeStats(searchContext, statsHolder -> { + statsHolder.fetchMetric.inc(tookInNanos); + statsHolder.fetchCurrent.dec(); + }); slowLogSearchService.onFetchPhase(searchContext, tookInNanos); } @@ -149,6 +136,15 @@ public final class ShardSearchStats { } } + private void computeStats(SearchContext searchContext, Consumer consumer) { + consumer.accept(totalStats); + if (searchContext.groupStats() != null) { + for (int i = 0; i < searchContext.groupStats().size(); i++) { + consumer.accept(groupStats(searchContext.groupStats().get(i))); + } + } + } + private StatsHolder groupStats(String group) { StatsHolder stats = groupsStats.get(group); if (stats == null) { @@ -184,26 +180,30 @@ public final class ShardSearchStats { public final MeanMetric queryMetric = new MeanMetric(); public final MeanMetric fetchMetric = new MeanMetric(); public final MeanMetric scrollMetric = new MeanMetric(); + public final MeanMetric suggestMetric = new MeanMetric(); public final CounterMetric queryCurrent = new CounterMetric(); public final CounterMetric fetchCurrent = new CounterMetric(); public final CounterMetric scrollCurrent = new CounterMetric(); + public final CounterMetric suggestCurrent = new CounterMetric(); public SearchStats.Stats stats() { return new SearchStats.Stats( queryMetric.count(), TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()), queryCurrent.count(), fetchMetric.count(), TimeUnit.NANOSECONDS.toMillis(fetchMetric.sum()), fetchCurrent.count(), - scrollMetric.count(), TimeUnit.NANOSECONDS.toMillis(scrollMetric.sum()), scrollCurrent.count() + scrollMetric.count(), TimeUnit.NANOSECONDS.toMillis(scrollMetric.sum()), scrollCurrent.count(), + suggestMetric.count(), TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()), suggestCurrent.count() ); } public long totalCurrent() { - return queryCurrent.count() + fetchCurrent.count() + scrollCurrent.count(); + return queryCurrent.count() + fetchCurrent.count() + scrollCurrent.count() + suggestCurrent.count(); } public void clear() { queryMetric.clear(); fetchMetric.clear(); scrollMetric.clear(); + suggestMetric.clear(); } } } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 7b19a12bbaa..b667a1de689 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -92,8 +92,6 @@ import org.elasticsearch.index.store.Store; import org.elasticsearch.index.store.Store.MetadataSnapshot; import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.index.suggest.stats.ShardSuggestMetric; -import org.elasticsearch.index.suggest.stats.SuggestStats; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; import org.elasticsearch.index.translog.TranslogStats; @@ -135,7 +133,6 @@ public class IndexShard extends AbstractIndexShardComponent { private final ShardRequestCache shardQueryCache; private final ShardFieldData shardFieldData; private final IndexFieldDataService indexFieldDataService; - private final ShardSuggestMetric shardSuggestMetric = new ShardSuggestMetric(); private final ShardBitsetFilterCache shardBitsetFilterCache; private final Object mutex = new Object(); private final String checkIndexOnStartup; @@ -254,10 +251,6 @@ public class IndexShard extends AbstractIndexShardComponent { return this.getService; } - public ShardSuggestMetric getSuggestMetric() { - return shardSuggestMetric; - } - public ShardBitsetFilterCache shardBitsetFilterCache() { return shardBitsetFilterCache; } @@ -631,10 +624,6 @@ public class IndexShard extends AbstractIndexShardComponent { return getEngine().getTranslog().stats(); } - public SuggestStats suggestStats() { - return shardSuggestMetric.stats(); - } - public CompletionStats completionStats(String... fields) { CompletionStats completionStats = new CompletionStats(); try (final Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) { diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java index edbebe8f033..865c0eb685b 100644 --- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java +++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java @@ -65,7 +65,7 @@ public final class SimilarityService extends AbstractIndexComponent { for (Map.Entry entry : similaritySettings.entrySet()) { String name = entry.getKey(); // Starting with v5.0 indices, it should no longer be possible to redefine built-in similarities - if(BUILT_IN.containsKey(name) && indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_0_0)) { + if(BUILT_IN.containsKey(name) && indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_0_0_alpha1)) { throw new IllegalArgumentException("Cannot redefine built-in Similarity [" + name + "]"); } Settings settings = entry.getValue(); @@ -83,7 +83,7 @@ public final class SimilarityService extends AbstractIndexComponent { } for (Map.Entry entry : addSimilarities(similaritySettings, DEFAULTS).entrySet()) { // Avoid overwriting custom providers for indices older that v5.0 - if (providers.containsKey(entry.getKey()) && indexSettings.getIndexVersionCreated().before(Version.V_5_0_0)) { + if (providers.containsKey(entry.getKey()) && indexSettings.getIndexVersionCreated().before(Version.V_5_0_0_alpha1)) { continue; } providers.put(entry.getKey(), entry.getValue()); diff --git a/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java b/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java deleted file mode 100644 index 750d7de7b22..00000000000 --- a/core/src/main/java/org/elasticsearch/index/suggest/stats/ShardSuggestMetric.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.suggest.stats; - -import org.elasticsearch.common.metrics.CounterMetric; -import org.elasticsearch.common.metrics.MeanMetric; - -import java.util.concurrent.TimeUnit; - -/** - * - */ -public final class ShardSuggestMetric { - private final MeanMetric suggestMetric = new MeanMetric(); - private final CounterMetric currentMetric = new CounterMetric(); - - /** - * Called before suggest - */ - public void preSuggest() { - currentMetric.inc(); - } - - /** - * Called after suggest - * @param tookInNanos time of suggest used in nanos - */ - public void postSuggest(long tookInNanos) { - currentMetric.dec(); - suggestMetric.inc(tookInNanos); - } - - /** - * @return The current stats - */ - public SuggestStats stats() { - return new SuggestStats(suggestMetric.count(), TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()), currentMetric.count()); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/suggest/stats/SuggestStats.java b/core/src/main/java/org/elasticsearch/index/suggest/stats/SuggestStats.java deleted file mode 100644 index 1183a64833a..00000000000 --- a/core/src/main/java/org/elasticsearch/index/suggest/stats/SuggestStats.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.suggest.stats; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentBuilderString; - -import java.io.IOException; - -/** - * Exposes suggest related statistics. - */ -public class SuggestStats implements Streamable, ToXContent { - - private long suggestCount; - private long suggestTimeInMillis; - private long current; - - public SuggestStats() { - } - - SuggestStats(long suggestCount, long suggestTimeInMillis, long current) { - this.suggestCount = suggestCount; - this.suggestTimeInMillis = suggestTimeInMillis; - this.current = current; - } - - /** - * @return The number of times the suggest api has been invoked. - */ - public long getCount() { - return suggestCount; - } - - /** - * @return The total amount of time spend in the suggest api - */ - public long getTimeInMillis() { - return suggestTimeInMillis; - } - - /** - * @return The total amount of time spend in the suggest api - */ - public TimeValue getTime() { - return new TimeValue(getTimeInMillis()); - } - - /** - * @return The total amount of active suggest api invocations. - */ - public long getCurrent() { - return current; - } - - public void add(SuggestStats suggestStats) { - if (suggestStats != null) { - suggestCount += suggestStats.getCount(); - suggestTimeInMillis += suggestStats.getTimeInMillis(); - current += suggestStats.getCurrent(); - } - } - - public static SuggestStats readSuggestStats(StreamInput in) throws IOException { - SuggestStats stats = new SuggestStats(); - stats.readFrom(in); - return stats; - } - - static final class Fields { - static final XContentBuilderString SUGGEST = new XContentBuilderString("suggest"); - static final XContentBuilderString TOTAL = new XContentBuilderString("total"); - static final XContentBuilderString TIME = new XContentBuilderString("time"); - static final XContentBuilderString TIME_IN_MILLIS = new XContentBuilderString("time_in_millis"); - static final XContentBuilderString CURRENT = new XContentBuilderString("current"); - } - - - @Override - public void readFrom(StreamInput in) throws IOException { - suggestCount = in.readVLong(); - suggestTimeInMillis = in.readVLong(); - current = in.readVLong(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(suggestCount); - out.writeVLong(suggestTimeInMillis); - out.writeVLong(current); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(Fields.SUGGEST); - builder.field(Fields.TOTAL, suggestCount); - builder.timeValueField(Fields.TIME_IN_MILLIS, Fields.TIME, suggestTimeInMillis); - builder.field(Fields.CURRENT, current); - builder.endObject(); - return builder; - } -} diff --git a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java index fce4e8411db..237975f2899 100644 --- a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java +++ b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java @@ -44,7 +44,6 @@ import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.index.suggest.stats.SuggestStats; import org.elasticsearch.search.suggest.completion.CompletionStats; import java.io.IOException; @@ -149,11 +148,6 @@ public class NodeIndicesStats implements Streamable, ToXContent { return stats.getSegments(); } - @Nullable - public SuggestStats getSuggest() { - return stats.getSuggest(); - } - @Nullable public RecoveryStats getRecoveryStats() { return stats.getRecoveryStats(); diff --git a/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java b/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java index a7d088ce214..5dab19581a3 100644 --- a/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java +++ b/core/src/main/java/org/elasticsearch/plugins/DummyPluginInfo.java @@ -20,9 +20,9 @@ package org.elasticsearch.plugins; public class DummyPluginInfo extends PluginInfo { - private DummyPluginInfo(String name, String description, String version, String classname, boolean isolated) { - super(name, description, version, classname, isolated); + private DummyPluginInfo(String name, String description, String version, String classname) { + super(name, description, version, classname); } - public static final DummyPluginInfo INSTANCE = new DummyPluginInfo("dummy_plugin_name", "dummy plugin description", "dummy_plugin_version", "DummyPluginName", true); + public static final DummyPluginInfo INSTANCE = new DummyPluginInfo("dummy_plugin_name", "dummy plugin description", "dummy_plugin_version", "DummyPluginName"); } diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index a260876db49..4096ad57885 100644 --- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -21,7 +21,6 @@ package org.elasticsearch.plugins; import joptsimple.OptionSet; import joptsimple.OptionSpec; -import org.apache.lucene.util.Constants; import org.apache.lucene.util.IOUtils; import org.elasticsearch.Build; import org.elasticsearch.Version; @@ -56,6 +55,7 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; +import java.util.Objects; import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -183,11 +183,18 @@ class InstallPluginCommand extends Command { final String version = Version.CURRENT.toString(); final String url; if (System.getProperty(PROPERTY_SUPPORT_STAGING_URLS, "false").equals("true")) { - url = String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/staging/%1$s-%2$s/org/elasticsearch/plugin/%3$s/%1$s/%3$s-%1$s.zip", - version, Build.CURRENT.shortHash(), pluginId); + url = String.format( + Locale.ROOT, + "https://download.elastic.co/elasticsearch/staging/%1$s-%2$s/org/elasticsearch/plugin/%3$s/%1$s/%3$s-%1$s.zip", + version, + Build.CURRENT.shortHash(), + pluginId); } else { - url = String.format(Locale.ROOT, "https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/%1$s/%2$s/%1$s-%2$s.zip", - pluginId, version); + url = String.format( + Locale.ROOT, + "https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/%1$s/%2$s/%1$s-%2$s.zip", + pluginId, + version); } terminal.println("-> Downloading " + pluginId + " from elastic"); return downloadZipAndChecksum(url, tmpDir); @@ -243,21 +250,8 @@ class InstallPluginCommand extends Command { private Path unzip(Path zip, Path pluginsDir) throws IOException, UserError { // unzip plugin to a staging temp dir - final Path target; - if (Constants.WINDOWS) { - target = Files.createTempDirectory(pluginsDir, ".installing-"); - } else { - Set perms = new HashSet<>(); - perms.add(PosixFilePermission.OWNER_EXECUTE); - perms.add(PosixFilePermission.OWNER_READ); - perms.add(PosixFilePermission.OWNER_WRITE); - perms.add(PosixFilePermission.GROUP_READ); - perms.add(PosixFilePermission.GROUP_EXECUTE); - perms.add(PosixFilePermission.OTHERS_READ); - perms.add(PosixFilePermission.OTHERS_EXECUTE); - target = Files.createTempDirectory(pluginsDir, ".installing-", PosixFilePermissions.asFileAttribute(perms)); - } - Files.createDirectories(target); + + final Path target = stagingDirectory(pluginsDir); boolean hasEsDir = false; // TODO: we should wrap this in a try/catch and try deleting the target dir on failure? @@ -302,6 +296,39 @@ class InstallPluginCommand extends Command { return target; } + private Path stagingDirectory(Path pluginsDir) throws IOException { + try { + Set perms = new HashSet<>(); + perms.add(PosixFilePermission.OWNER_EXECUTE); + perms.add(PosixFilePermission.OWNER_READ); + perms.add(PosixFilePermission.OWNER_WRITE); + perms.add(PosixFilePermission.GROUP_READ); + perms.add(PosixFilePermission.GROUP_EXECUTE); + perms.add(PosixFilePermission.OTHERS_READ); + perms.add(PosixFilePermission.OTHERS_EXECUTE); + return Files.createTempDirectory(pluginsDir, ".installing-", PosixFilePermissions.asFileAttribute(perms)); + } catch (IllegalArgumentException e) { + // Jimfs throws an IAE where it should throw an UOE + // remove when google/jimfs#30 is integrated into Jimfs + // and the Jimfs test dependency is upgraded to include + // this pull request + final StackTraceElement[] elements = e.getStackTrace(); + if (elements.length >= 1 && + elements[0].getClassName().equals("com.google.common.jimfs.AttributeService") && + elements[0].getMethodName().equals("setAttributeInternal")) { + return stagingDirectoryWithoutPosixPermissions(pluginsDir); + } else { + throw e; + } + } catch (UnsupportedOperationException e) { + return stagingDirectoryWithoutPosixPermissions(pluginsDir); + } + } + + private Path stagingDirectoryWithoutPosixPermissions(Path pluginsDir) throws IOException { + return Files.createTempDirectory(pluginsDir, ".installing-"); + } + /** Load information about the plugin, and verify it can be installed with no errors. */ private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch) throws Exception { // read and validate the plugin descriptor @@ -315,7 +342,7 @@ class InstallPluginCommand extends Command { } // check for jar hell before any copying - jarHellCheck(pluginRoot, env.pluginsFile(), info.isIsolated()); + jarHellCheck(pluginRoot, env.pluginsFile()); // read optional security policy (extra permissions) // if it exists, confirm or warn the user @@ -328,19 +355,13 @@ class InstallPluginCommand extends Command { } /** check a candidate plugin for jar hell before installing it */ - private void jarHellCheck(Path candidate, Path pluginsDir, boolean isolated) throws Exception { + void jarHellCheck(Path candidate, Path pluginsDir) throws Exception { // create list of current jars in classpath final List jars = new ArrayList<>(); jars.addAll(Arrays.asList(JarHell.parseClassPath())); // read existing bundles. this does some checks on the installation too. - List bundles = PluginsService.getPluginBundles(pluginsDir); - - // if we aren't isolated, we need to jarhellcheck against any other non-isolated plugins - // that's always the first bundle - if (isolated == false) { - jars.addAll(bundles.get(0).urls); - } + PluginsService.getPluginBundles(pluginsDir); // add plugin jars to the list Path pluginJars[] = FileSystemUtils.files(candidate, "*.jar"); @@ -367,7 +388,10 @@ class InstallPluginCommand extends Command { final Path destination = env.pluginsFile().resolve(info.getName()); if (Files.exists(destination)) { - throw new UserError(ExitCodes.USAGE, "plugin directory " + destination.toAbsolutePath() + " already exists. To update the plugin, uninstall it first using 'remove " + info.getName() + "' command"); + throw new UserError( + ExitCodes.USAGE, + "plugin directory " + destination.toAbsolutePath() + + " already exists. To update the plugin, uninstall it first using 'remove " + info.getName() + "' command"); } Path tmpBinDir = tmpRoot.resolve("bin"); @@ -404,30 +428,30 @@ class InstallPluginCommand extends Command { } Files.createDirectory(destBinDir); - Set perms = new HashSet<>(); - if (Constants.WINDOWS == false) { - // setup file attributes for the installed files to those of the parent dir - PosixFileAttributeView binAttrs = Files.getFileAttributeView(destBinDir.getParent(), PosixFileAttributeView.class); - if (binAttrs != null) { - perms = new HashSet<>(binAttrs.readAttributes().permissions()); - // setting execute bits, since this just means "the file is executable", and actual execution requires read - perms.add(PosixFilePermission.OWNER_EXECUTE); - perms.add(PosixFilePermission.GROUP_EXECUTE); - perms.add(PosixFilePermission.OTHERS_EXECUTE); - } + // setup file attributes for the installed files to those of the parent dir + final Set perms = new HashSet<>(); + final PosixFileAttributeView binAttributeView = Files.getFileAttributeView(destBinDir.getParent(), PosixFileAttributeView.class); + if (binAttributeView != null) { + perms.addAll(binAttributeView.readAttributes().permissions()); + // setting execute bits, since this just means "the file is executable", and actual execution requires read + perms.add(PosixFilePermission.OWNER_EXECUTE); + perms.add(PosixFilePermission.GROUP_EXECUTE); + perms.add(PosixFilePermission.OTHERS_EXECUTE); } try (DirectoryStream stream = Files.newDirectoryStream(tmpBinDir)) { for (Path srcFile : stream) { if (Files.isDirectory(srcFile)) { - throw new UserError(ExitCodes.DATA_ERROR, "Directories not allowed in bin dir for plugin " + info.getName() + ", found " + srcFile.getFileName()); + throw new UserError( + ExitCodes.DATA_ERROR, + "Directories not allowed in bin dir for plugin " + info.getName() + ", found " + srcFile.getFileName()); } Path destFile = destBinDir.resolve(tmpBinDir.relativize(srcFile)); Files.copy(srcFile, destFile); - if (perms.isEmpty() == false) { - PosixFileAttributeView view = Files.getFileAttributeView(destFile, PosixFileAttributeView.class); + final PosixFileAttributeView view = Files.getFileAttributeView(destFile, PosixFileAttributeView.class); + if (view != null) { view.setPermissions(perms); } } @@ -446,15 +470,12 @@ class InstallPluginCommand extends Command { // create the plugin's config dir "if necessary" Files.createDirectories(destConfigDir); - - final PosixFileAttributes destConfigDirAttributes; - if (Constants.WINDOWS) { - destConfigDirAttributes = null; - } else { - destConfigDirAttributes = - Files.getFileAttributeView(destConfigDir.getParent(), PosixFileAttributeView.class).readAttributes(); + final PosixFileAttributeView destConfigDirAttributesView = + Files.getFileAttributeView(destConfigDir.getParent(), PosixFileAttributeView.class); + final PosixFileAttributes destConfigDirAttributes = + destConfigDirAttributesView != null ? destConfigDirAttributesView.readAttributes() : null; + if (destConfigDirAttributes != null) { setOwnerGroup(destConfigDir, destConfigDirAttributes); - } try (DirectoryStream stream = Files.newDirectoryStream(tmpConfigDir)) { @@ -466,7 +487,7 @@ class InstallPluginCommand extends Command { Path destFile = destConfigDir.resolve(tmpConfigDir.relativize(srcFile)); if (Files.exists(destFile) == false) { Files.copy(srcFile, destFile); - if (Constants.WINDOWS == false) { + if (destConfigDirAttributes != null) { setOwnerGroup(destFile, destConfigDirAttributes); } } @@ -475,8 +496,10 @@ class InstallPluginCommand extends Command { IOUtils.rm(tmpConfigDir); // clean up what we just copied } - private static void setOwnerGroup(Path path, PosixFileAttributes attributes) throws IOException { + private static void setOwnerGroup(final Path path, final PosixFileAttributes attributes) throws IOException { + Objects.requireNonNull(attributes); PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(path, PosixFileAttributeView.class); + assert fileAttributeView != null; fileAttributeView.setOwner(attributes.owner()); fileAttributeView.setGroup(attributes.group()); } diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java index 73464d054dd..bd063b3312c 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginInfo.java @@ -44,14 +44,12 @@ public class PluginInfo implements Streamable, ToXContent { static final XContentBuilderString URL = new XContentBuilderString("url"); static final XContentBuilderString VERSION = new XContentBuilderString("version"); static final XContentBuilderString CLASSNAME = new XContentBuilderString("classname"); - static final XContentBuilderString ISOLATED = new XContentBuilderString("isolated"); } private String name; private String description; private String version; private String classname; - private boolean isolated; public PluginInfo() { } @@ -63,12 +61,11 @@ public class PluginInfo implements Streamable, ToXContent { * @param description Its description * @param version Version number */ - PluginInfo(String name, String description, String version, String classname, boolean isolated) { + PluginInfo(String name, String description, String version, String classname) { this.name = name; this.description = description; this.version = version; this.classname = classname; - this.isolated = isolated; } /** reads (and validates) plugin metadata descriptor file */ @@ -106,13 +103,12 @@ public class PluginInfo implements Streamable, ToXContent { } JarHell.checkVersionFormat(javaVersionString); JarHell.checkJavaVersion(name, javaVersionString); - boolean isolated = Boolean.parseBoolean(props.getProperty("isolated", "true")); String classname = props.getProperty("classname"); if (classname == null) { throw new IllegalArgumentException("Property [classname] is missing for plugin [" + name + "]"); } - return new PluginInfo(name, description, version, classname, isolated); + return new PluginInfo(name, description, version, classname); } /** @@ -129,13 +125,6 @@ public class PluginInfo implements Streamable, ToXContent { return description; } - /** - * @return true if plugin has isolated classloader - */ - public boolean isIsolated() { - return isolated; - } - /** * @return plugin's classname */ @@ -162,7 +151,6 @@ public class PluginInfo implements Streamable, ToXContent { this.description = in.readString(); this.version = in.readString(); this.classname = in.readString(); - this.isolated = in.readBoolean(); } @Override @@ -171,7 +159,6 @@ public class PluginInfo implements Streamable, ToXContent { out.writeString(description); out.writeString(version); out.writeString(classname); - out.writeBoolean(isolated); } @Override @@ -181,7 +168,6 @@ public class PluginInfo implements Streamable, ToXContent { builder.field(Fields.VERSION, version); builder.field(Fields.DESCRIPTION, description); builder.field(Fields.CLASSNAME, classname); - builder.field(Fields.ISOLATED, isolated); builder.endObject(); return builder; @@ -212,8 +198,7 @@ public class PluginInfo implements Streamable, ToXContent { .append("Name: ").append(name).append("\n") .append("Description: ").append(description).append("\n") .append("Version: ").append(version).append("\n") - .append(" * Classname: ").append(classname).append("\n") - .append(" * Isolated: ").append(isolated); + .append(" * Classname: ").append(classname); return information.toString(); } diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java index cf953cd1529..c1f3043cd32 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -103,7 +103,7 @@ public class PluginsService extends AbstractComponent { // first we load plugins that are on the classpath. this is for tests and transport clients for (Class pluginClass : classpathPlugins) { Plugin plugin = loadPlugin(pluginClass, settings); - PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), "NA", pluginClass.getName(), false); + PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), "NA", pluginClass.getName()); if (logger.isTraceEnabled()) { logger.trace("plugin loaded from classpath [{}]", pluginInfo); } @@ -302,9 +302,6 @@ public class PluginsService extends AbstractComponent { continue; // skip over .DS_Store etc } PluginInfo info = PluginInfo.readFromProperties(module); - if (!info.isIsolated()) { - throw new IllegalStateException("modules must be isolated: " + info); - } Bundle bundle = new Bundle(); bundle.plugins.add(info); // gather urls for jar files @@ -329,8 +326,6 @@ public class PluginsService extends AbstractComponent { } List bundles = new ArrayList<>(); - // a special purgatory for plugins that directly depend on each other - bundles.add(new Bundle()); try (DirectoryStream stream = Files.newDirectoryStream(pluginsDirectory)) { for (Path plugin : stream) { @@ -354,13 +349,8 @@ public class PluginsService extends AbstractComponent { urls.add(jar.toRealPath().toUri().toURL()); } } - final Bundle bundle; - if (info.isIsolated() == false) { - bundle = bundles.get(0); // purgatory - } else { - bundle = new Bundle(); - bundles.add(bundle); - } + final Bundle bundle = new Bundle(); + bundles.add(bundle); bundle.plugins.add(info); bundle.urls.addAll(urls); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java index 9a9d1991298..168d091968e 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; @@ -51,6 +52,7 @@ public class RestListTasksAction extends BaseRestHandler { String[] actions = Strings.splitStringByCommaToArray(request.param("actions")); TaskId parentTaskId = new TaskId(request.param("parent_task_id")); boolean waitForCompletion = request.paramAsBoolean("wait_for_completion", false); + TimeValue timeout = request.paramAsTime("timeout", null); ListTasksRequest listTasksRequest = new ListTasksRequest(); listTasksRequest.setTaskId(taskId); @@ -59,6 +61,7 @@ public class RestListTasksAction extends BaseRestHandler { listTasksRequest.setActions(actions); listTasksRequest.setParentTaskId(parentTaskId); listTasksRequest.setWaitForCompletion(waitForCompletion); + listTasksRequest.setTimeout(timeout); client.admin().cluster().listTasks(listTasksRequest, new RestToXContentListener<>(channel)); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java index 92fb21db38c..dbda83709ba 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java @@ -71,7 +71,7 @@ public class RestIndicesStatsAction extends BaseRestHandler { indicesStatsRequest.docs(metrics.contains("docs")); indicesStatsRequest.store(metrics.contains("store")); indicesStatsRequest.indexing(metrics.contains("indexing")); - indicesStatsRequest.search(metrics.contains("search")); + indicesStatsRequest.search(metrics.contains("search") || metrics.contains("suggest")); indicesStatsRequest.get(metrics.contains("get")); indicesStatsRequest.merge(metrics.contains("merge")); indicesStatsRequest.refresh(metrics.contains("refresh")); @@ -82,7 +82,6 @@ public class RestIndicesStatsAction extends BaseRestHandler { indicesStatsRequest.segments(metrics.contains("segments")); indicesStatsRequest.fieldData(metrics.contains("fielddata")); indicesStatsRequest.completion(metrics.contains("completion")); - indicesStatsRequest.suggest(metrics.contains("suggest")); indicesStatsRequest.requestCache(metrics.contains("request_cache")); indicesStatsRequest.recovery(metrics.contains("recovery")); indicesStatsRequest.translog(metrics.contains("translog")); diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java index 398ef9c15f7..958fa40b54b 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java @@ -490,14 +490,14 @@ public class RestIndicesAction extends AbstractCatAction { table.addCell(indexStats == null ? null : indexStats.getTotal().getWarmer().totalTime()); table.addCell(indexStats == null ? null : indexStats.getPrimaries().getWarmer().totalTime()); - table.addCell(indexStats == null ? null : indexStats.getTotal().getSuggest().getCurrent()); - table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSuggest().getCurrent()); + table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestCurrent()); + table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestCurrent()); - table.addCell(indexStats == null ? null : indexStats.getTotal().getSuggest().getTime()); - table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSuggest().getTime()); + table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestTime()); + table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestTime()); - table.addCell(indexStats == null ? null : indexStats.getTotal().getSuggest().getCount()); - table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSuggest().getCount()); + table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getSuggestCount()); + table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getSuggestCount()); table.addCell(indexStats == null ? null : indexStats.getTotal().getTotalMemory()); table.addCell(indexStats == null ? null : indexStats.getPrimaries().getTotalMemory()); diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java index c7b1550cbae..54ee855b543 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java @@ -49,7 +49,6 @@ import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.percolator.PercolatorQueryCacheStats; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; -import org.elasticsearch.index.suggest.stats.SuggestStats; import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.monitor.jvm.JvmInfo; @@ -362,10 +361,9 @@ public class RestNodesAction extends AbstractCatAction { table.addCell(segmentsStats == null ? null : segmentsStats.getVersionMapMemory()); table.addCell(segmentsStats == null ? null : segmentsStats.getBitsetMemory()); - SuggestStats suggestStats = indicesStats == null ? null : indicesStats.getSuggest(); - table.addCell(suggestStats == null ? null : suggestStats.getCurrent()); - table.addCell(suggestStats == null ? null : suggestStats.getTime()); - table.addCell(suggestStats == null ? null : suggestStats.getCount()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestCurrent()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestTime()); + table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestCount()); table.endRow(); } diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java index a0812f3e9a4..b4ba4bbeffa 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java @@ -66,7 +66,6 @@ public class RestThreadPoolAction extends AbstractCatAction { ThreadPool.Names.REFRESH, ThreadPool.Names.SEARCH, ThreadPool.Names.SNAPSHOT, - ThreadPool.Names.SUGGEST, ThreadPool.Names.WARMER }; @@ -81,7 +80,6 @@ public class RestThreadPoolAction extends AbstractCatAction { "r", "s", "sn", - "su", "w" }; diff --git a/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java b/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java index 53d9e668de1..1bbc662929c 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java @@ -19,14 +19,15 @@ package org.elasticsearch.rest.action.suggest; -import org.elasticsearch.action.suggest.SuggestRequest; -import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; @@ -41,6 +42,7 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestBuilderListener; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.Suggesters; @@ -73,28 +75,29 @@ public class RestSuggestAction extends BaseRestHandler { @Override public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException { - SuggestRequest suggestRequest = new SuggestRequest(Strings.splitStringByCommaToArray(request.param("index"))); - suggestRequest.indicesOptions(IndicesOptions.fromRequest(request, suggestRequest.indicesOptions())); + final SearchRequest searchRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")), new SearchSourceBuilder()); + searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); if (RestActions.hasBodyContent(request)) { final BytesReference sourceBytes = RestActions.getRestContent(request); try (XContentParser parser = XContentFactory.xContent(sourceBytes).createParser(sourceBytes)) { final QueryParseContext context = new QueryParseContext(queryRegistry); context.reset(parser); context.parseFieldMatcher(parseFieldMatcher); - suggestRequest.suggest(SuggestBuilder.fromXContent(context, suggesters)); + searchRequest.source().suggest(SuggestBuilder.fromXContent(context, suggesters)); } } else { throw new IllegalArgumentException("no content or source provided to execute suggestion"); } - suggestRequest.routing(request.param("routing")); - suggestRequest.preference(request.param("preference")); - - client.suggest(suggestRequest, new RestBuilderListener(channel) { + searchRequest.routing(request.param("routing")); + searchRequest.preference(request.param("preference")); + client.search(searchRequest, new RestBuilderListener(channel) { @Override - public RestResponse buildResponse(SuggestResponse response, XContentBuilder builder) throws Exception { - RestStatus restStatus = RestStatus.status(response.getSuccessfulShards(), response.getTotalShards(), response.getShardFailures()); + public RestResponse buildResponse(SearchResponse response, XContentBuilder builder) throws Exception { + RestStatus restStatus = RestStatus.status(response.getSuccessfulShards(), + response.getTotalShards(), response.getShardFailures()); builder.startObject(); - buildBroadcastShardsHeader(builder, request, response); + buildBroadcastShardsHeader(builder, request, response.getTotalShards(), + response.getSuccessfulShards(), response.getFailedShards(), response.getShardFailures()); Suggest suggest = response.getSuggest(); if (suggest != null) { suggest.toInnerXContent(builder, request); diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java index ae6d5aaf4be..96db4b11469 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java @@ -226,6 +226,11 @@ import org.elasticsearch.search.highlight.Highlighters; import org.elasticsearch.search.query.QueryPhase; import org.elasticsearch.search.rescore.QueryRescorerBuilder; import org.elasticsearch.search.rescore.RescoreBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.GeoDistanceSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.ScriptSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.search.suggest.SuggestionBuilder; @@ -346,6 +351,7 @@ public class SearchModule extends AbstractModule { configureFetchSubPhase(); configureShapes(); configureRescorers(); + configureSorts(); } protected void configureFetchSubPhase() { @@ -489,6 +495,13 @@ public class SearchModule extends AbstractModule { namedWriteableRegistry.registerPrototype(RescoreBuilder.class, QueryRescorerBuilder.PROTOTYPE); } + private void configureSorts() { + namedWriteableRegistry.registerPrototype(SortBuilder.class, GeoDistanceSortBuilder.PROTOTYPE); + namedWriteableRegistry.registerPrototype(SortBuilder.class, ScoreSortBuilder.PROTOTYPE); + namedWriteableRegistry.registerPrototype(SortBuilder.class, ScriptSortBuilder.PROTOTYPE); + namedWriteableRegistry.registerPrototype(SortBuilder.class, FieldSortBuilder.PROTOTYPE); + } + private void registerBuiltinFunctionScoreParsers() { registerFunctionScoreParser(new ScriptScoreFunctionParser()); registerFunctionScoreParser(new GaussDecayFunctionParser()); diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index a1b7f93d0d1..98085bfed35 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -21,6 +21,7 @@ package org.elasticsearch.search; import com.carrotsearch.hppc.ObjectFloatHashMap; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.cache.recycler.PageCacheRecycler; @@ -41,7 +42,6 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.ConcurrentMapLong; import org.elasticsearch.common.util.concurrent.FutureUtils; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; @@ -92,6 +92,7 @@ import org.elasticsearch.search.query.QuerySearchResultProvider; import org.elasticsearch.search.query.ScrollQuerySearchResult; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.searchafter.SearchAfterBuilder; +import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.threadpool.ThreadPool; @@ -99,6 +100,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicLong; @@ -683,33 +685,13 @@ public class SearchService extends AbstractLifecycleComponent imp context.parsedPostFilter(queryShardContext.toQuery(source.postFilter())); } if (source.sorts() != null) { - XContentParser completeSortParser = null; try { - XContentBuilder completeSortBuilder = XContentFactory.jsonBuilder(); - completeSortBuilder.startObject(); - completeSortBuilder.startArray("sort"); - for (BytesReference sort : source.sorts()) { - XContentParser parser = XContentFactory.xContent(sort).createParser(sort); - parser.nextToken(); - completeSortBuilder.copyCurrentStructure(parser); + Optional optionalSort = SortBuilder.buildSort(source.sorts(), context.getQueryShardContext()); + if (optionalSort.isPresent()) { + context.sort(optionalSort.get()); } - completeSortBuilder.endArray(); - completeSortBuilder.endObject(); - BytesReference completeSortBytes = completeSortBuilder.bytes(); - completeSortParser = XContentFactory.xContent(completeSortBytes).createParser(completeSortBytes); - completeSortParser.nextToken(); - completeSortParser.nextToken(); - completeSortParser.nextToken(); - this.elementParsers.get("sort").parse(completeSortParser, context); - } catch (Exception e) { - String sSource = "_na_"; - try { - sSource = source.toString(); - } catch (Throwable e1) { - // ignore - } - XContentLocation location = completeSortParser != null ? completeSortParser.getTokenLocation() : null; - throw new SearchParseException(context, "failed to parse sort source [" + sSource + "]", location, e); + } catch (IOException e) { + throw new SearchContextException(context, "failed to create sort elements", e); } } context.trackScores(source.trackScores()); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java index 9f5f3e443ef..fd60fa93857 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java @@ -21,13 +21,9 @@ package org.elasticsearch.search.aggregations.metrics.tophits; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.AggregationInitializationException; import org.elasticsearch.search.aggregations.AggregatorBuilder; @@ -38,6 +34,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -57,7 +54,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sorts = null; + private List> sorts = null; private HighlightBuilder highlightBuilder; private List fieldNames; private List fieldDataFields; @@ -119,6 +116,9 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sort) { if (sort == null) { throw new IllegalArgumentException("[sort] must not be null: [" + name + "]"); } - try { - if (sorts == null) { + if (sorts == null) { sorts = new ArrayList<>(); - } - // NORELEASE when sort has been refactored and made writeable - // add the sortBuilcer to the List directly instead of - // serialising to XContent - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - sort.toXContent(builder, EMPTY_PARAMS); - builder.endObject(); - sorts.add(builder.bytes()); - } catch (IOException e) { - throw new RuntimeException(e); } + sorts.add(sort); return this; } /** * Adds a sort builder. */ - public TopHitsAggregatorBuilder sorts(List sorts) { + public TopHitsAggregatorBuilder sorts(List> sorts) { if (sorts == null) { throw new IllegalArgumentException("[sorts] must not be null: [" + name + "]"); } if (this.sorts == null) { this.sorts = new ArrayList<>(); } - for (BytesReference sort : sorts) { + for (SortBuilder sort : sorts) { this.sorts.add(sort); } return this; @@ -181,7 +173,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sorts() { + public List> sorts() { return sorts; } @@ -509,10 +501,8 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sort : sorts) { + sort.toXContent(builder, params); } builder.endArray(); } @@ -562,9 +552,9 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sorts = new ArrayList<>(); + List> sorts = new ArrayList<>(); for (int i = 0; i < size; i++) { - sorts.add(in.readBytesReference()); + sorts.add(in.readSortBuilder()); } factory.sorts = sorts; } @@ -612,8 +602,8 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder sort : sorts) { + out.writeSortBuilder(sort); } } out.writeBoolean(trackScores); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java index c4d2165e308..be10f4ce878 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java @@ -19,12 +19,7 @@ package org.elasticsearch.search.aggregations.metrics.tophits; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.common.xcontent.XContentParser; +import org.apache.lucene.search.Sort; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.aggregations.Aggregator; @@ -35,27 +30,27 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext; -import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsContext.FieldDataField; +import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SubSearchContext; -import org.elasticsearch.search.sort.SortParseElement; +import org.elasticsearch.search.sort.SortBuilder; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; public class TopHitsAggregatorFactory extends AggregatorFactory { - private static final SortParseElement sortParseElement = new SortParseElement(); private final int from; private final int size; private final boolean explain; private final boolean version; private final boolean trackScores; - private final List sorts; + private final List> sorts; private final HighlightBuilder highlightBuilder; private final List fieldNames; private final List fieldDataFields; @@ -63,7 +58,7 @@ public class TopHitsAggregatorFactory extends AggregatorFactory sorts, HighlightBuilder highlightBuilder, List fieldNames, List fieldDataFields, + List> sorts, HighlightBuilder highlightBuilder, List fieldNames, List fieldDataFields, List scriptFields, FetchSourceContext fetchSourceContext, AggregationContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, Map metaData) throws IOException { super(name, type, context, parent, subFactories, metaData); @@ -90,27 +85,9 @@ public class TopHitsAggregatorFactory extends AggregatorFactory optionalSort = SortBuilder.buildSort(sorts, subSearchContext.getQueryShardContext()); + if (optionalSort.isPresent()) { + subSearchContext.sort(optionalSort.get()); } } if (fieldNames != null) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java index 03921d620ee..9ec6b46dafe 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsParser.java @@ -19,9 +19,6 @@ package org.elasticsearch.search.aggregations.metrics.tophits; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.script.Script; @@ -30,6 +27,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; +import org.elasticsearch.search.sort.SortBuilder; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -124,9 +123,7 @@ public class TopHitsParser implements Aggregator.Parser { } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder.HIGHLIGHT_FIELD)) { factory.highlighter(HighlightBuilder.PROTOTYPE.fromXContent(context)); } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) { - List sorts = new ArrayList<>(); - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); - sorts.add(xContentBuilder.bytes()); + List> sorts = SortBuilder.fromXContent(context); factory.sorts(sorts); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", @@ -157,11 +154,7 @@ public class TopHitsParser implements Aggregator.Parser { } factory.fieldDataFields(fieldDataFields); } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) { - List sorts = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); - sorts.add(xContentBuilder.bytes()); - } + List> sorts = SortBuilder.fromXContent(context); factory.sorts(sorts); } else if (context.parseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) { factory.fetchSource(FetchSourceContext.parse(parser, context)); diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index a20ec535238..c83794ced90 100644 --- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -52,6 +52,7 @@ import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.searchafter.SearchAfterBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -139,7 +140,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ private Boolean version; - private List sorts; + private List> sorts; private boolean trackScores = false; @@ -336,6 +337,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ * The sort ordering */ public SearchSourceBuilder sort(String name, SortOrder order) { + if (name.equals(ScoreSortBuilder.NAME)) { + return sort(SortBuilders.scoreSort().order(order)); + } return sort(SortBuilders.fieldSort(name).order(order)); } @@ -346,32 +350,27 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ * The name of the field to sort by */ public SearchSourceBuilder sort(String name) { + if (name.equals(ScoreSortBuilder.NAME)) { + return sort(SortBuilders.scoreSort()); + } return sort(SortBuilders.fieldSort(name)); } /** * Adds a sort builder. */ - public SearchSourceBuilder sort(SortBuilder sort) { - try { + public SearchSourceBuilder sort(SortBuilder sort) { if (sorts == null) { sorts = new ArrayList<>(); } - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - sort.toXContent(builder, EMPTY_PARAMS); - builder.endObject(); - sorts.add(builder.bytes()); + sorts.add(sort); return this; - } catch (IOException e) { - throw new RuntimeException(e); - } } /** * Gets the bytes representing the sort builders for this request. */ - public List sorts() { + public List> sorts() { return sorts; } @@ -730,6 +729,14 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ return ext; } + /** + * @return true if the source only has suggest + */ + public boolean isSuggestOnly() { + return suggestBuilder != null + && queryBuilder == null && aggregations == null; + } + /** * Rewrites this search source builder into its primitive form. e.g. by * rewriting the QueryBuilder. If the builder did not change the identity @@ -907,9 +914,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } else if (context.parseFieldMatcher().match(currentFieldName, SUGGEST_FIELD)) { suggestBuilder = SuggestBuilder.fromXContent(context, suggesters); } else if (context.parseFieldMatcher().match(currentFieldName, SORT_FIELD)) { - sorts = new ArrayList<>(); - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); - sorts.add(xContentBuilder.bytes()); + sorts = new ArrayList<>(SortBuilder.fromXContent(context)); } else if (context.parseFieldMatcher().match(currentFieldName, EXT_FIELD)) { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); ext = xContentBuilder.bytes(); @@ -940,11 +945,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ } } } else if (context.parseFieldMatcher().match(currentFieldName, SORT_FIELD)) { - sorts = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser); - sorts.add(xContentBuilder.bytes()); - } + sorts = new ArrayList<>(SortBuilder.fromXContent(context)); } else if (context.parseFieldMatcher().match(currentFieldName, RESCORE_FIELD)) { rescoreBuilders = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { @@ -1057,10 +1058,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ if (sorts != null) { builder.startArray(SORT_FIELD.getPreferredName()); - for (BytesReference sort : sorts) { - XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(sort); - parser.nextToken(); - builder.copyCurrentStructure(parser); + for (SortBuilder sort : sorts) { + sort.toXContent(builder, params); } builder.endArray(); } @@ -1266,9 +1265,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ builder.size = in.readVInt(); if (in.readBoolean()) { int size = in.readVInt(); - List sorts = new ArrayList<>(); + List> sorts = new ArrayList<>(); for (int i = 0; i < size; i++) { - sorts.add(in.readBytesReference()); + sorts.add(in.readSortBuilder()); } builder.sorts = sorts; } @@ -1382,8 +1381,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ out.writeBoolean(hasSorts); if (hasSorts) { out.writeVInt(sorts.size()); - for (BytesReference sort : sorts) { - out.writeBytesReference(sort); + for (SortBuilder sort : sorts) { + out.writeSortBuilder(sort); } } boolean hasStats = stats != null; diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java index 39c432f9cc9..f9cf3f09f39 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java @@ -35,7 +35,6 @@ import org.elasticsearch.search.highlight.HighlighterParseElement; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.InternalSearchHits; import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.search.sort.SortParseElement; import java.io.IOException; import java.util.HashMap; @@ -51,8 +50,8 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase { private FetchPhase fetchPhase; @Inject - public InnerHitsFetchSubPhase(SortParseElement sortParseElement, FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) { - parseElements = singletonMap("inner_hits", new InnerHitsParseElement(sortParseElement, sourceParseElement, highlighterParseElement, + public InnerHitsFetchSubPhase(FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) { + parseElements = singletonMap("inner_hits", new InnerHitsParseElement(sourceParseElement, highlighterParseElement, fieldDataFieldsParseElement, scriptFieldsParseElement)); } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java index 077268ac52f..fc1b2bf399f 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java @@ -32,7 +32,6 @@ import org.elasticsearch.search.fetch.source.FetchSourceParseElement; import org.elasticsearch.search.highlight.HighlighterParseElement; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.SubSearchContext; -import org.elasticsearch.search.sort.SortParseElement; import java.util.HashMap; import java.util.Map; @@ -43,14 +42,12 @@ import static org.elasticsearch.index.query.support.InnerHitsQueryParserHelper.p */ public class InnerHitsParseElement implements SearchParseElement { - private final SortParseElement sortParseElement; private final FetchSourceParseElement sourceParseElement; private final HighlighterParseElement highlighterParseElement; private final FieldDataFieldsParseElement fieldDataFieldsParseElement; private final ScriptFieldsParseElement scriptFieldsParseElement; - public InnerHitsParseElement(SortParseElement sortParseElement, FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) { - this.sortParseElement = sortParseElement; + public InnerHitsParseElement(FetchSourceParseElement sourceParseElement, HighlighterParseElement highlighterParseElement, FieldDataFieldsParseElement fieldDataFieldsParseElement, ScriptFieldsParseElement scriptFieldsParseElement) { this.sourceParseElement = sourceParseElement; this.highlighterParseElement = highlighterParseElement; this.fieldDataFieldsParseElement = fieldDataFieldsParseElement; @@ -184,10 +181,10 @@ public class InnerHitsParseElement implements SearchParseElement { } else if ("inner_hits".equals(fieldName)) { childInnerHits = parseInnerHits(parser, context, searchContext); } else { - parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sortParseElement, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); + parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); } } else { - parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sortParseElement, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); + parseCommonInnerHitOptions(parser, token, fieldName, subSearchContext, sourceParseElement, highlighterParseElement, scriptFieldsParseElement, fieldDataFieldsParseElement); } } diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java index 71a289331f8..f3fe48f6682 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java @@ -191,6 +191,9 @@ public class DefaultSearchContext extends SearchContext { */ @Override public void preProcess() { + if (hasOnlySuggest() ) { + return; + } if (scrollContext == null) { long from = from() == -1 ? 0 : from(); long size = size() == -1 ? 10 : size(); diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 1881109c9b6..f9d0b6b9283 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -353,6 +353,14 @@ public abstract class SearchContext implements Releasable { } } + /** + * @return true if the request contains only suggest + */ + public final boolean hasOnlySuggest() { + return request().source() != null + && request().source().isSuggestOnly(); + } + /** * Looks up the given field, but does not restrict to fields in the types set on this context. */ diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 5a98744505a..62210655a00 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -58,7 +58,6 @@ import org.elasticsearch.search.profile.ProfileShardResult; import org.elasticsearch.search.profile.Profiler; import org.elasticsearch.search.rescore.RescorePhase; import org.elasticsearch.search.rescore.RescoreSearchContext; -import org.elasticsearch.search.sort.SortParseElement; import org.elasticsearch.search.sort.TrackScoresParseElement; import org.elasticsearch.search.suggest.SuggestPhase; @@ -98,7 +97,6 @@ public class QueryPhase implements SearchPhase { parseElements.put("query", new QueryParseElement()); parseElements.put("post_filter", new PostFilterParseElement()); parseElements.put("postFilter", new PostFilterParseElement()); - parseElements.put("sort", new SortParseElement()); parseElements.put("trackScores", new TrackScoresParseElement()); parseElements.put("track_scores", new TrackScoresParseElement()); parseElements.put("min_score", new MinScoreParseElement()); @@ -118,6 +116,12 @@ public class QueryPhase implements SearchPhase { @Override public void execute(SearchContext searchContext) throws QueryPhaseExecutionException { + if (searchContext.hasOnlySuggest()) { + suggestPhase.execute(searchContext); + // TODO: fix this once we can fetch docs for suggestions + searchContext.queryResult().topDocs(new TopDocs(0, Lucene.EMPTY_SCORE_DOCS, 0)); + return; + } // Pre-process aggregations as late as possible. In the case of a DFS_Q_T_F // request, preProcess is called on the DFS phase phase, this is why we pre-process them // here to make sure it happens during the QUERY phase diff --git a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index 8ff9ac0b754..f75ea37f854 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -20,12 +20,10 @@ package org.elasticsearch.search.sort; import org.apache.lucene.search.SortField; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -44,7 +42,7 @@ import java.util.Objects; * A sort builder to sort based on a document field. */ public class FieldSortBuilder extends SortBuilder { - static final FieldSortBuilder PROTOTYPE = new FieldSortBuilder(""); + public static final FieldSortBuilder PROTOTYPE = new FieldSortBuilder("_na_"); public static final String NAME = "field_sort"; public static final ParseField NESTED_PATH = new ParseField("nested_path"); public static final ParseField NESTED_FILTER = new ParseField("nested_filter"); @@ -108,19 +106,12 @@ public class FieldSortBuilder extends SortBuilder { * _first to sort missing last or first respectively. */ public FieldSortBuilder missing(Object missing) { - if (missing instanceof String) { - this.missing = BytesRefs.toBytesRef(missing); - } else { - this.missing = missing; - } + this.missing = missing; return this; } /** Returns the value used when a field is missing in a doc. */ public Object missing() { - if (missing instanceof BytesRef) { - return ((BytesRef) missing).utf8ToString(); - } return missing; } @@ -207,14 +198,11 @@ public class FieldSortBuilder extends SortBuilder { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.startObject(fieldName); builder.field(ORDER_FIELD.getPreferredName(), order); if (missing != null) { - if (missing instanceof BytesRef) { - builder.field(MISSING.getPreferredName(), ((BytesRef) missing).utf8ToString()); - } else { - builder.field(MISSING.getPreferredName(), missing); - } + builder.field(MISSING.getPreferredName(), missing); } if (unmappedType != null) { builder.field(UNMAPPED_TYPE.getPreferredName(), unmappedType); @@ -229,6 +217,7 @@ public class FieldSortBuilder extends SortBuilder { builder.field(NESTED_PATH.getPreferredName(), nestedPath); } builder.endObject(); + builder.endObject(); return builder; } @@ -375,7 +364,7 @@ public class FieldSortBuilder extends SortBuilder { if (context.parseFieldMatcher().match(currentFieldName, NESTED_PATH)) { nestedPath = parser.text(); } else if (context.parseFieldMatcher().match(currentFieldName, MISSING)) { - missing = parser.objectBytes(); + missing = parser.objectText(); } else if (context.parseFieldMatcher().match(currentFieldName, ORDER)) { String sortOrder = parser.text(); if ("asc".equals(sortOrder)) { diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index 0b1e52c0aa6..4712799c658 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -73,7 +73,7 @@ public class GeoDistanceSortBuilder extends SortBuilder public static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path"); public static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter"); - static final GeoDistanceSortBuilder PROTOTYPE = new GeoDistanceSortBuilder("", -1, -1); + public static final GeoDistanceSortBuilder PROTOTYPE = new GeoDistanceSortBuilder("_na_", -1, -1); private final String fieldName; private final List points = new ArrayList<>(); @@ -299,6 +299,7 @@ public class GeoDistanceSortBuilder extends SortBuilder @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.startObject(NAME); builder.startArray(fieldName); @@ -324,6 +325,7 @@ public class GeoDistanceSortBuilder extends SortBuilder builder.field(COERCE_FIELD.getPreferredName(), coerce); builder.field(IGNORE_MALFORMED_FIELD.getPreferredName(), ignoreMalformed); + builder.endObject(); builder.endObject(); return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java deleted file mode 100644 index aff0e68fc1d..00000000000 --- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.sort; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.join.BitSetProducer; -import org.apache.lucene.util.BitSet; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.Version; -import org.elasticsearch.common.geo.GeoDistance; -import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.GeoUtils; -import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; -import org.elasticsearch.index.fielddata.NumericDoubleValues; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.support.NestedInnerQueryParseSupport; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * - */ -public class GeoDistanceSortParser implements SortParser { - - @Override - public String[] names() { - return new String[]{"_geo_distance", "_geoDistance"}; - } - - @Override - public SortField parse(XContentParser parser, QueryShardContext context) throws IOException { - String fieldName = null; - List geoPoints = new ArrayList<>(); - DistanceUnit unit = DistanceUnit.DEFAULT; - GeoDistance geoDistance = GeoDistance.DEFAULT; - boolean reverse = false; - MultiValueMode sortMode = null; - NestedInnerQueryParseSupport nestedHelper = null; - - final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0); - boolean coerce = GeoDistanceSortBuilder.DEFAULT_COERCE; - boolean ignoreMalformed = GeoDistanceSortBuilder.DEFAULT_IGNORE_MALFORMED; - - XContentParser.Token token; - String currentName = parser.currentName(); - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentName = parser.currentName(); - } else if (token == XContentParser.Token.START_ARRAY) { - GeoDistanceSortBuilder.parseGeoPoints(parser, geoPoints); - - fieldName = currentName; - } else if (token == XContentParser.Token.START_OBJECT) { - // the json in the format of -> field : { lat : 30, lon : 12 } - if ("nested_filter".equals(currentName) || "nestedFilter".equals(currentName)) { - if (nestedHelper == null) { - nestedHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedHelper.filter(); - } else { - fieldName = currentName; - GeoPoint point = new GeoPoint(); - GeoUtils.parseGeoPoint(parser, point); - geoPoints.add(point); - } - } else if (token.isValue()) { - if ("reverse".equals(currentName)) { - reverse = parser.booleanValue(); - } else if ("order".equals(currentName)) { - reverse = "desc".equals(parser.text()); - } else if (currentName.equals("unit")) { - unit = DistanceUnit.fromString(parser.text()); - } else if (currentName.equals("distance_type") || currentName.equals("distanceType")) { - geoDistance = GeoDistance.fromString(parser.text()); - } else if ("coerce".equals(currentName) || (indexCreatedBeforeV2_0 && "normalize".equals(currentName))) { - coerce = parser.booleanValue(); - if (coerce == true) { - ignoreMalformed = true; - } - } else if ("ignore_malformed".equals(currentName)) { - boolean ignoreMalformedFlag = parser.booleanValue(); - if (coerce == false) { - ignoreMalformed = ignoreMalformedFlag; - } - } else if ("sort_mode".equals(currentName) || "sortMode".equals(currentName) || "mode".equals(currentName)) { - sortMode = MultiValueMode.fromString(parser.text()); - } else if ("nested_path".equals(currentName) || "nestedPath".equals(currentName)) { - if (nestedHelper == null) { - nestedHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedHelper.setPath(parser.text()); - } else { - GeoPoint point = new GeoPoint(); - point.resetFromString(parser.text()); - geoPoints.add(point); - fieldName = currentName; - } - } - } - - // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes - if (!indexCreatedBeforeV2_0 && !ignoreMalformed) { - for (GeoPoint point : geoPoints) { - if (point.lat() > 90.0 || point.lat() < -90.0) { - throw new ElasticsearchParseException("illegal latitude value [{}] for [GeoDistanceSort]", point.lat()); - } - if (point.lon() > 180.0 || point.lon() < -180) { - throw new ElasticsearchParseException("illegal longitude value [{}] for [GeoDistanceSort]", point.lon()); - } - } - } - - if (coerce) { - for (GeoPoint point : geoPoints) { - GeoUtils.normalizePoint(point, coerce, coerce); - } - } - - if (sortMode == null) { - sortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN; - } - - if (sortMode == MultiValueMode.SUM) { - throw new IllegalArgumentException("sort_mode [sum] isn't supported for sorting by geo distance"); - } - - MappedFieldType fieldType = context.fieldMapper(fieldName); - if (fieldType == null) { - throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort"); - } - final MultiValueMode finalSortMode = sortMode; // final reference for use in the anonymous class - final IndexGeoPointFieldData geoIndexFieldData = context.getForField(fieldType); - final FixedSourceDistance[] distances = new FixedSourceDistance[geoPoints.size()]; - for (int i = 0; i< geoPoints.size(); i++) { - distances[i] = geoDistance.fixedSourceDistance(geoPoints.get(i).lat(), geoPoints.get(i).lon(), unit); - } - - final Nested nested; - if (nestedHelper != null && nestedHelper.getPath() != null) { - BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter()); - Query innerDocumentsQuery; - if (nestedHelper.filterFound()) { - // TODO: use queries instead - innerDocumentsQuery = nestedHelper.getInnerFilter(); - } else { - innerDocumentsQuery = nestedHelper.getNestedObjectMapper().nestedTypeFilter(); - } - - nested = new Nested(rootDocumentsFilter, innerDocumentsQuery); - } else { - nested = null; - } - - IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() { - - @Override - public SortField.Type reducedType() { - return SortField.Type.DOUBLE; - } - - @Override - public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - return new FieldComparator.DoubleComparator(numHits, null, null) { - @Override - protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException { - final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues(); - final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(geoPointValues, distances); - final NumericDoubleValues selectedValues; - if (nested == null) { - selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE); - } else { - final BitSet rootDocs = nested.rootDocs(context); - final DocIdSetIterator innerDocs = nested.innerDocs(context); - selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE, rootDocs, innerDocs, context.reader().maxDoc()); - } - return selectedValues.getRawDoubleValues(); - } - }; - } - - }; - - return new SortField(fieldName, geoDistanceComparatorSource, reverse); - } - -} diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java index 3d273f6fe24..9124540f881 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java @@ -39,7 +39,7 @@ import java.util.Objects; public class ScoreSortBuilder extends SortBuilder { public static final String NAME = "_score"; - static final ScoreSortBuilder PROTOTYPE = new ScoreSortBuilder(); + public static final ScoreSortBuilder PROTOTYPE = new ScoreSortBuilder(); public static final ParseField ORDER_FIELD = new ParseField("order"); private static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE); private static final SortField SORT_SCORE_REVERSE = new SortField(null, SortField.Type.SCORE, true); @@ -52,9 +52,11 @@ public class ScoreSortBuilder extends SortBuilder { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); builder.startObject(NAME); builder.field(ORDER_FIELD.getPreferredName(), order); builder.endObject(); + builder.endObject(); return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index b905af881b3..b79eb6e2147 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -67,7 +67,7 @@ import java.util.Objects; public class ScriptSortBuilder extends SortBuilder { public static final String NAME = "_script"; - static final ScriptSortBuilder PROTOTYPE = new ScriptSortBuilder(new Script("_na_"), ScriptSortType.STRING); + public static final ScriptSortBuilder PROTOTYPE = new ScriptSortBuilder(new Script("_na_"), ScriptSortType.STRING); public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField SCRIPT_FIELD = new ParseField("script"); public static final ParseField SORTMODE_FIELD = new ParseField("mode"); @@ -179,6 +179,7 @@ public class ScriptSortBuilder extends SortBuilder { @Override public XContentBuilder toXContent(XContentBuilder builder, Params builderParams) throws IOException { + builder.startObject(); builder.startObject(NAME); builder.field(SCRIPT_FIELD.getPreferredName(), script); builder.field(TYPE_FIELD.getPreferredName(), type); @@ -193,6 +194,7 @@ public class ScriptSortBuilder extends SortBuilder { builder.field(NESTED_FILTER_FIELD.getPreferredName(), nestedFilter, builderParams); } builder.endObject(); + builder.endObject(); return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java deleted file mode 100644 index af8531da87c..00000000000 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.sort; - -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.join.BitSetProducer; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.FieldData; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.NumericDoubleValues; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; -import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.support.NestedInnerQueryParseSupport; -import org.elasticsearch.script.LeafSearchScript; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.Script.ScriptField; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptParameterParser; -import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue; -import org.elasticsearch.script.SearchScript; -import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * - */ -public class ScriptSortParser implements SortParser { - - @Override - public String[] names() { - return new String[]{"_script"}; - } - - @Override - public SortField parse(XContentParser parser, QueryShardContext context) throws IOException { - ScriptParameterParser scriptParameterParser = new ScriptParameterParser(); - Script script = null; - ScriptSortType type = null; - Map params = null; - boolean reverse = false; - MultiValueMode sortMode = null; - NestedInnerQueryParseSupport nestedHelper = null; - - XContentParser.Token token; - String currentName = parser.currentName(); - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if (context.parseFieldMatcher().match(currentName, ScriptField.SCRIPT)) { - script = Script.parse(parser, context.parseFieldMatcher()); - } else if ("params".equals(currentName)) { - params = parser.map(); - } else if ("nested_filter".equals(currentName) || "nestedFilter".equals(currentName)) { - if (nestedHelper == null) { - nestedHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedHelper.filter(); - } - } else if (token.isValue()) { - if ("reverse".equals(currentName)) { - reverse = parser.booleanValue(); - } else if ("order".equals(currentName)) { - reverse = "desc".equals(parser.text()); - } else if (scriptParameterParser.token(currentName, token, parser, context.parseFieldMatcher())) { - // Do Nothing (handled by ScriptParameterParser - } else if ("type".equals(currentName)) { - type = ScriptSortType.fromString(parser.text()); - } else if ("mode".equals(currentName)) { - sortMode = MultiValueMode.fromString(parser.text()); - } else if ("nested_path".equals(currentName) || "nestedPath".equals(currentName)) { - if (nestedHelper == null) { - nestedHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedHelper.setPath(parser.text()); - } - } - } - - if (script == null) { // Didn't find anything using the new API so try using the old one instead - ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue(); - if (scriptValue != null) { - if (params == null) { - params = new HashMap<>(); - } - script = new Script(scriptValue.script(), scriptValue.scriptType(), scriptParameterParser.lang(), params); - } - } else if (params != null) { - throw new ParsingException(parser.getTokenLocation(), "script params must be specified inside script object"); - } - - if (script == null) { - throw new ParsingException(parser.getTokenLocation(), "_script sorting requires setting the script to sort by"); - } - if (type == null) { - throw new ParsingException(parser.getTokenLocation(), "_script sorting requires setting the type of the script"); - } - final SearchScript searchScript = context.getScriptService().search( - context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap()); - - if (ScriptSortType.STRING.equals(type) && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) { - throw new ParsingException(parser.getTokenLocation(), "type [string] doesn't support mode [" + sortMode + "]"); - } - - if (sortMode == null) { - sortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN; - } - - final Nested nested; - if (nestedHelper != null && nestedHelper.getPath() != null) { - BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter()); - Query innerDocumentsFilter; - if (nestedHelper.filterFound()) { - // TODO: use queries instead - innerDocumentsFilter = nestedHelper.getInnerFilter(); - } else { - innerDocumentsFilter = nestedHelper.getNestedObjectMapper().nestedTypeFilter(); - } - nested = new Nested(rootDocumentsFilter, innerDocumentsFilter); - } else { - nested = null; - } - - final IndexFieldData.XFieldComparatorSource fieldComparatorSource; - switch (type) { - case STRING: - fieldComparatorSource = new BytesRefFieldComparatorSource(null, null, sortMode, nested) { - LeafSearchScript leafScript; - @Override - protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOException { - leafScript = searchScript.getLeafSearchScript(context); - final BinaryDocValues values = new BinaryDocValues() { - final BytesRefBuilder spare = new BytesRefBuilder(); - @Override - public BytesRef get(int docID) { - leafScript.setDocument(docID); - spare.copyChars(leafScript.run().toString()); - return spare.get(); - } - }; - return FieldData.singleton(values, null); - } - @Override - protected void setScorer(Scorer scorer) { - leafScript.setScorer(scorer); - } - }; - break; - case NUMBER: - fieldComparatorSource = new DoubleValuesComparatorSource(null, Double.MAX_VALUE, sortMode, nested) { - LeafSearchScript leafScript; - @Override - protected SortedNumericDoubleValues getValues(LeafReaderContext context) throws IOException { - leafScript = searchScript.getLeafSearchScript(context); - final NumericDoubleValues values = new NumericDoubleValues() { - @Override - public double get(int docID) { - leafScript.setDocument(docID); - return leafScript.runAsDouble(); - } - }; - return FieldData.singleton(values, null); - } - @Override - protected void setScorer(Scorer scorer) { - leafScript.setScorer(scorer); - } - }; - break; - default: - throw new ParsingException(parser.getTokenLocation(), "custom script sort type [" + type + "] not supported"); - } - - return new SortField("_script", fieldComparatorSource, reverse); - } -} diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java index c966a343751..e007ac7736e 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.sort; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.join.BitSetProducer; import org.elasticsearch.action.support.ToXContentToBytes; @@ -34,6 +35,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.ArrayList; @@ -41,6 +43,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import static java.util.Collections.unmodifiableMap; @@ -157,6 +160,41 @@ public abstract class SortBuilder> extends ToXContentTo } } + public static void parseSort(XContentParser parser, SearchContext context) throws IOException { + QueryParseContext parseContext = context.getQueryShardContext().parseContext(); + parseContext.reset(parser); + Optional sortOptional = buildSort(SortBuilder.fromXContent(parseContext), context.getQueryShardContext()); + if (sortOptional.isPresent()) { + context.sort(sortOptional.get()); + } + } + + public static Optional buildSort(List> sortBuilders, QueryShardContext context) throws IOException { + List sortFields = new ArrayList<>(sortBuilders.size()); + for (SortBuilder builder : sortBuilders) { + sortFields.add(builder.build(context)); + } + if (!sortFields.isEmpty()) { + // optimize if we just sort on score non reversed, we don't really + // need sorting + boolean sort; + if (sortFields.size() > 1) { + sort = true; + } else { + SortField sortField = sortFields.get(0); + if (sortField.getType() == SortField.Type.SCORE && !sortField.getReverse()) { + sort = false; + } else { + sort = true; + } + } + if (sort) { + return Optional.of(new Sort(sortFields.toArray(new SortField[sortFields.size()]))); + } + } + return Optional.empty(); + } + protected static Nested resolveNested(QueryShardContext context, String nestedPath, QueryBuilder nestedFilter) throws IOException { Nested nested = null; if (nestedPath != null) { diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java b/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java deleted file mode 100644 index 1ed2a457a5f..00000000000 --- a/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.sort; - -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.join.BitSetProducer; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.QueryShardException; -import org.elasticsearch.index.query.support.NestedInnerQueryParseSupport; -import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.search.SearchParseElement; -import org.elasticsearch.search.internal.SearchContext; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static java.util.Collections.unmodifiableMap; - -/** - * - */ -public class SortParseElement implements SearchParseElement { - - private static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE); - private static final SortField SORT_SCORE_REVERSE = new SortField(null, SortField.Type.SCORE, true); - private static final SortField SORT_DOC = new SortField(null, SortField.Type.DOC); - private static final SortField SORT_DOC_REVERSE = new SortField(null, SortField.Type.DOC, true); - - public static final ParseField UNMAPPED_TYPE = new ParseField("unmapped_type"); - - public static final String SCORE_FIELD_NAME = "_score"; - public static final String DOC_FIELD_NAME = "_doc"; - - private static final Map PARSERS; - - static { - Map parsers = new HashMap<>(); - addParser(parsers, new ScriptSortParser()); - addParser(parsers, new GeoDistanceSortParser()); - PARSERS = unmodifiableMap(parsers); - } - - private static void addParser(Map parsers, SortParser parser) { - for (String name : parser.names()) { - parsers.put(name, parser); - } - } - - @Override - public void parse(XContentParser parser, SearchContext context) throws IOException { - List sortFields = parse(parser, context.getQueryShardContext()); - if (!sortFields.isEmpty()) { - // optimize if we just sort on score non reversed, we don't really need sorting - boolean sort; - if (sortFields.size() > 1) { - sort = true; - } else { - SortField sortField = sortFields.get(0); - if (sortField.getType() == SortField.Type.SCORE && !sortField.getReverse()) { - sort = false; - } else { - sort = true; - } - } - if (sort) { - context.sort(new Sort(sortFields.toArray(new SortField[sortFields.size()]))); - } - } - } - - List parse(XContentParser parser, QueryShardContext context) throws IOException { - XContentParser.Token token = parser.currentToken(); - List sortFields = new ArrayList<>(2); - if (token == XContentParser.Token.START_ARRAY) { - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - if (token == XContentParser.Token.START_OBJECT) { - addCompoundSortField(parser, context, sortFields); - } else if (token == XContentParser.Token.VALUE_STRING) { - addSortField(context, sortFields, parser.text(), false, null, null, null, null); - } else { - throw new IllegalArgumentException("malformed sort format, within the sort array, an object, or an actual string are allowed"); - } - } - } else if (token == XContentParser.Token.VALUE_STRING) { - addSortField(context, sortFields, parser.text(), false, null, null, null, null); - } else if (token == XContentParser.Token.START_OBJECT) { - addCompoundSortField(parser, context, sortFields); - } else { - throw new IllegalArgumentException("malformed sort format, either start with array, object, or an actual string"); - } - return sortFields; - } - - private void addCompoundSortField(XContentParser parser, QueryShardContext context, List sortFields) throws IOException { - XContentParser.Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - String fieldName = parser.currentName(); - boolean reverse = false; - String missing = null; - String innerJsonName = null; - String unmappedType = null; - MultiValueMode sortMode = null; - NestedInnerQueryParseSupport nestedFilterParseHelper = null; - token = parser.nextToken(); - if (token == XContentParser.Token.VALUE_STRING) { - String direction = parser.text(); - if (direction.equals("asc")) { - reverse = SCORE_FIELD_NAME.equals(fieldName); - } else if (direction.equals("desc")) { - reverse = !SCORE_FIELD_NAME.equals(fieldName); - } else { - throw new IllegalArgumentException("sort direction [" + fieldName + "] not supported"); - } - addSortField(context, sortFields, fieldName, reverse, unmappedType, missing, sortMode, nestedFilterParseHelper); - } else { - if (PARSERS.containsKey(fieldName)) { - sortFields.add(PARSERS.get(fieldName).parse(parser, context)); - } else { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - innerJsonName = parser.currentName(); - } else if (token.isValue()) { - if ("reverse".equals(innerJsonName)) { - reverse = parser.booleanValue(); - } else if ("order".equals(innerJsonName)) { - if ("asc".equals(parser.text())) { - reverse = SCORE_FIELD_NAME.equals(fieldName); - } else if ("desc".equals(parser.text())) { - reverse = !SCORE_FIELD_NAME.equals(fieldName); - } - } else if ("missing".equals(innerJsonName)) { - missing = parser.textOrNull(); - } else if (context.parseFieldMatcher().match(innerJsonName, UNMAPPED_TYPE)) { - unmappedType = parser.textOrNull(); - } else if ("mode".equals(innerJsonName)) { - sortMode = MultiValueMode.fromString(parser.text()); - } else if ("nested_path".equals(innerJsonName) || "nestedPath".equals(innerJsonName)) { - if (nestedFilterParseHelper == null) { - nestedFilterParseHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedFilterParseHelper.setPath(parser.text()); - } else { - throw new IllegalArgumentException("sort option [" + innerJsonName + "] not supported"); - } - } else if (token == XContentParser.Token.START_OBJECT) { - if ("nested_filter".equals(innerJsonName) || "nestedFilter".equals(innerJsonName)) { - if (nestedFilterParseHelper == null) { - nestedFilterParseHelper = new NestedInnerQueryParseSupport(parser, context); - } - nestedFilterParseHelper.filter(); - } else { - throw new IllegalArgumentException("sort option [" + innerJsonName + "] not supported"); - } - } - } - addSortField(context, sortFields, fieldName, reverse, unmappedType, missing, sortMode, nestedFilterParseHelper); - } - } - } - } - } - - private void addSortField(QueryShardContext context, List sortFields, String fieldName, boolean reverse, String unmappedType, @Nullable final String missing, MultiValueMode sortMode, NestedInnerQueryParseSupport nestedHelper) throws IOException { - if (SCORE_FIELD_NAME.equals(fieldName)) { - if (reverse) { - sortFields.add(SORT_SCORE_REVERSE); - } else { - sortFields.add(SORT_SCORE); - } - } else if (DOC_FIELD_NAME.equals(fieldName)) { - if (reverse) { - sortFields.add(SORT_DOC_REVERSE); - } else { - sortFields.add(SORT_DOC); - } - } else { - MappedFieldType fieldType = context.fieldMapper(fieldName); - if (fieldType == null) { - if (unmappedType != null) { - fieldType = context.getMapperService().unmappedFieldType(unmappedType); - } else { - throw new QueryShardException(context, "No mapping found for [" + fieldName + "] in order to sort on"); - } - } - - if (!fieldType.isSortable()) { - throw new QueryShardException(context, "Sorting not supported for field[" + fieldName + "]"); - } - - // We only support AVG and SUM on number based fields - if (fieldType.isNumeric() == false && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) { - sortMode = null; - } - if (sortMode == null) { - sortMode = resolveDefaultSortMode(reverse); - } - - final Nested nested; - if (nestedHelper != null && nestedHelper.getPath() != null) { - BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter()); - Query innerDocumentsQuery; - if (nestedHelper.filterFound()) { - innerDocumentsQuery = nestedHelper.getInnerFilter(); - } else { - innerDocumentsQuery = nestedHelper.getNestedObjectMapper().nestedTypeFilter(); - } - nested = new Nested(rootDocumentsFilter, innerDocumentsQuery); - } else { - nested = null; - } - - IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.getForField(fieldType) - .comparatorSource(missing, sortMode, nested); - sortFields.add(new SortField(fieldType.name(), fieldComparatorSource, reverse)); - } - } - - private static MultiValueMode resolveDefaultSortMode(boolean reverse) { - return reverse ? MultiValueMode.MAX : MultiValueMode.MIN; - } - -} diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortParser.java b/core/src/main/java/org/elasticsearch/search/sort/SortParser.java deleted file mode 100644 index 519d9adb957..00000000000 --- a/core/src/main/java/org/elasticsearch/search/sort/SortParser.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.sort; - -import org.apache.lucene.search.SortField; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.QueryShardContext; - -import java.io.IOException; - -/** - * - */ -public interface SortParser { - - String[] names(); - - SortField parse(XContentParser parser, QueryShardContext context) throws IOException; -} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java index 8831d5d093a..520322587bb 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.search.suggest; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.component.AbstractComponent; @@ -65,10 +64,6 @@ public class SuggestPhase extends AbstractComponent implements SearchPhase { if (suggest == null) { return; } - context.queryResult().suggest(execute(suggest, context.searcher())); - } - - public Suggest execute(SuggestionSearchContext suggest, IndexSearcher searcher) { try { CharsRefBuilder spare = new CharsRefBuilder(); final List>> suggestions = new ArrayList<>(suggest.suggestions().size()); @@ -76,14 +71,14 @@ public class SuggestPhase extends AbstractComponent implements SearchPhase { for (Map.Entry entry : suggest.suggestions().entrySet()) { SuggestionSearchContext.SuggestionContext suggestion = entry.getValue(); Suggester suggester = suggestion.getSuggester(); - Suggestion> result = suggester.execute(entry.getKey(), suggestion, searcher, spare); + Suggestion> result = + suggester.execute(entry.getKey(), suggestion, context.searcher(), spare); if (result != null) { assert entry.getKey().equals(result.name); suggestions.add(result); } } - - return new Suggest(suggestions); + context.queryResult().suggest(new Suggest(suggestions)); } catch (IOException e) { throw new ElasticsearchException("I/O exception during suggest phase", e); } diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskId.java b/core/src/main/java/org/elasticsearch/tasks/TaskId.java index 5c5ad36cc17..d1f29e65226 100644 --- a/core/src/main/java/org/elasticsearch/tasks/TaskId.java +++ b/core/src/main/java/org/elasticsearch/tasks/TaskId.java @@ -73,15 +73,15 @@ public final class TaskId implements Writeable { } public boolean isSet() { - return id == -1L; + return id != -1L; } @Override public String toString() { if (isSet()) { - return "unset"; - } else { return nodeId + ":" + id; + } else { + return "unset"; } } diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java index 0c785573c99..c881e1756e8 100644 --- a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java +++ b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java @@ -76,7 +76,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen CancellableTaskHolder oldHolder = cancellableTasks.put(task.getId(), holder); assert oldHolder == null; // Check if this task was banned before we start it - if (task.getParentTaskId().isSet() == false && banedParents.isEmpty() == false) { + if (task.getParentTaskId().isSet() && banedParents.isEmpty() == false) { String reason = banedParents.get(task.getParentTaskId()); if (reason != null) { try { @@ -241,7 +241,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen CancellableTaskHolder holder = taskEntry.getValue(); CancellableTask task = holder.getTask(); TaskId parentTaskId = task.getParentTaskId(); - if (parentTaskId.isSet() == false && lastDiscoveryNodes.nodeExists(parentTaskId.getNodeId()) == false) { + if (parentTaskId.isSet() && lastDiscoveryNodes.nodeExists(parentTaskId.getNodeId()) == false) { if (task.cancelOnParentLeaving()) { holder.cancel("Coordinating node [" + parentTaskId.getNodeId() + "] left the cluster"); } diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index a04594e706a..b223b477740 100644 --- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -82,7 +82,6 @@ public class ThreadPool extends AbstractComponent implements Closeable { public static final String INDEX = "index"; public static final String BULK = "bulk"; public static final String SEARCH = "search"; - public static final String SUGGEST = "suggest"; public static final String MANAGEMENT = "management"; public static final String FLUSH = "flush"; public static final String REFRESH = "refresh"; @@ -139,7 +138,6 @@ public class ThreadPool extends AbstractComponent implements Closeable { map.put(Names.INDEX, ThreadPoolType.FIXED); map.put(Names.BULK, ThreadPoolType.FIXED); map.put(Names.SEARCH, ThreadPoolType.FIXED); - map.put(Names.SUGGEST, ThreadPoolType.FIXED); map.put(Names.MANAGEMENT, ThreadPoolType.SCALING); map.put(Names.FLUSH, ThreadPoolType.SCALING); map.put(Names.REFRESH, ThreadPoolType.SCALING); @@ -227,7 +225,6 @@ public class ThreadPool extends AbstractComponent implements Closeable { add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.BULK).size(availableProcessors).queueSize(50)); add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.GET).size(availableProcessors).queueSize(1000)); add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.SEARCH).size(((availableProcessors * 3) / 2) + 1).queueSize(1000)); - add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.SUGGEST).size(availableProcessors).queueSize(1000)); add(defaultExecutorTypeSettings, new ExecutorSettingsBuilder(Names.MANAGEMENT).size(5).keepAlive("5m")); // no queue as this means clients will need to handle rejections on listener queue even if the operation succeeded // the assumption here is that the listeners should be very lightweight on the listeners side diff --git a/core/src/test/java/org/elasticsearch/VersionTests.java b/core/src/test/java/org/elasticsearch/VersionTests.java index 17a41c30275..65c91f5daab 100644 --- a/core/src/test/java/org/elasticsearch/VersionTests.java +++ b/core/src/test/java/org/elasticsearch/VersionTests.java @@ -32,7 +32,7 @@ import java.util.Locale; import java.util.Map; import static org.elasticsearch.Version.V_2_2_0; -import static org.elasticsearch.Version.V_5_0_0; +import static org.elasticsearch.Version.V_5_0_0_alpha1; import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.containsString; @@ -42,21 +42,21 @@ import static org.hamcrest.Matchers.sameInstance; public class VersionTests extends ESTestCase { public void testVersionComparison() throws Exception { - assertThat(V_2_2_0.before(V_5_0_0), is(true)); + assertThat(V_2_2_0.before(V_5_0_0_alpha1), is(true)); assertThat(V_2_2_0.before(V_2_2_0), is(false)); - assertThat(V_5_0_0.before(V_2_2_0), is(false)); + assertThat(V_5_0_0_alpha1.before(V_2_2_0), is(false)); - assertThat(V_2_2_0.onOrBefore(V_5_0_0), is(true)); + assertThat(V_2_2_0.onOrBefore(V_5_0_0_alpha1), is(true)); assertThat(V_2_2_0.onOrBefore(V_2_2_0), is(true)); - assertThat(V_5_0_0.onOrBefore(V_2_2_0), is(false)); + assertThat(V_5_0_0_alpha1.onOrBefore(V_2_2_0), is(false)); - assertThat(V_2_2_0.after(V_5_0_0), is(false)); + assertThat(V_2_2_0.after(V_5_0_0_alpha1), is(false)); assertThat(V_2_2_0.after(V_2_2_0), is(false)); - assertThat(V_5_0_0.after(V_2_2_0), is(true)); + assertThat(V_5_0_0_alpha1.after(V_2_2_0), is(true)); - assertThat(V_2_2_0.onOrAfter(V_5_0_0), is(false)); + assertThat(V_2_2_0.onOrAfter(V_5_0_0_alpha1), is(false)); assertThat(V_2_2_0.onOrAfter(V_2_2_0), is(true)); - assertThat(V_5_0_0.onOrAfter(V_2_2_0), is(true)); + assertThat(V_5_0_0_alpha1.onOrAfter(V_2_2_0), is(true)); assertTrue(Version.fromString("5.0.0-alpha2").onOrAfter(Version.fromString("5.0.0-alpha1"))); assertTrue(Version.fromString("5.0.0").onOrAfter(Version.fromString("5.0.0-beta2"))); @@ -133,7 +133,7 @@ public class VersionTests extends ESTestCase { public void testIndexCreatedVersion() { // an actual index has a IndexMetaData.SETTING_INDEX_UUID - final Version version = randomFrom(Version.V_2_0_0, Version.V_2_3_0, Version.V_5_0_0); + final Version version = randomFrom(Version.V_2_0_0, Version.V_2_3_0, Version.V_5_0_0_alpha1); assertEquals(version, Version.indexCreated(Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "foo").put(IndexMetaData.SETTING_VERSION_CREATED, version).build())); } @@ -142,13 +142,13 @@ public class VersionTests extends ESTestCase { assertThat(Version.V_2_1_0.minimumCompatibilityVersion(), equalTo(Version.V_2_0_0)); assertThat(Version.V_2_2_0.minimumCompatibilityVersion(), equalTo(Version.V_2_0_0)); assertThat(Version.V_2_3_0.minimumCompatibilityVersion(), equalTo(Version.V_2_0_0)); - assertThat(Version.V_5_0_0.minimumCompatibilityVersion(), equalTo(Version.V_5_0_0)); + assertThat(Version.V_5_0_0_alpha1.minimumCompatibilityVersion(), equalTo(Version.V_5_0_0_alpha1)); } public void testToString() { // with 2.0.beta we lowercase assertEquals("2.0.0-beta1", Version.V_2_0_0_beta1.toString()); - assertEquals("5.0.0", Version.V_5_0_0.toString()); + assertEquals("5.0.0-alpha1", Version.V_5_0_0_alpha1.toString()); assertEquals("2.3.0", Version.V_2_3_0.toString()); assertEquals("0.90.0.Beta1", Version.fromString("0.90.0.Beta1").toString()); assertEquals("1.0.0.Beta1", Version.fromString("1.0.0.Beta1").toString()); @@ -240,6 +240,8 @@ public class VersionTests extends ESTestCase { number = number.replace("-beta", "_beta"); } else if (v.isRC()) { number = number.replace("-rc", "_rc"); + } else if (v.isAlpha()) { + number = number.replace("-alpha", "_alpha"); } assertEquals("V_" + number.replace('.', '_'), constantName); } else { diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index b22d93ef6b2..e9ade3ef5af 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.action.admin.cluster.node.tasks; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; @@ -59,6 +60,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; +import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.emptyCollectionOf; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -118,9 +120,9 @@ public class TasksIT extends ESIntegTestCase { // Verify that one of these tasks is a parent of another task if (tasks.get(0).getParentTaskId().isSet()) { - assertParentTask(Collections.singletonList(tasks.get(1)), tasks.get(0)); - } else { assertParentTask(Collections.singletonList(tasks.get(0)), tasks.get(1)); + } else { + assertParentTask(Collections.singletonList(tasks.get(1)), tasks.get(0)); } } @@ -406,6 +408,17 @@ public class TasksIT extends ESIntegTestCase { assertThat(waitResponseFuture.get().getTasks(), emptyCollectionOf(TaskInfo.class)); } + public void testTasksWaitForAllTask() throws Exception { + // Spin up a request to wait for all tasks in the cluster to make sure it doesn't cause an infinite loop + ListTasksResponse response = client().admin().cluster().prepareListTasks().setWaitForCompletion(true) + .setTimeout(timeValueSeconds(10)).get(); + + // It should finish quickly and without complaint and list the list tasks themselves + assertThat(response.getNodeFailures(), emptyCollectionOf(FailedNodeException.class)); + assertThat(response.getTaskFailures(), emptyCollectionOf(TaskOperationFailure.class)); + assertThat(response.getTasks().size(), greaterThanOrEqualTo(1)); + } + @Override public void tearDown() throws Exception { for (Map.Entry, RecordingTaskManagerListener> entry : listeners.entrySet()) { @@ -474,7 +487,7 @@ public class TasksIT extends ESIntegTestCase { */ private void assertParentTask(List tasks, TaskInfo parentTask) { for (TaskInfo task : tasks) { - assertFalse(task.getParentTaskId().isSet()); + assertTrue(task.getParentTaskId().isSet()); assertEquals(parentTask.getNode().getId(), task.getParentTaskId().getNodeId()); assertTrue(Strings.hasLength(task.getParentTaskId().getNodeId())); assertEquals(parentTask.getId(), task.getParentTaskId().getId()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index 64d69a4864f..4b478b52bd0 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -604,7 +604,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase { @Override protected TestTaskResponse taskOperation(TestTasksRequest request, Task task) { logger.info("Task action on node {}", node); - if (failTaskOnNode == node && task.getParentTaskId().isSet() == false) { + if (failTaskOnNode == node && task.getParentTaskId().isSet()) { logger.info("Failing on node {}", node); throw new RuntimeException("Task level failure"); } diff --git a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java index 51274af9a01..632646146fe 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java @@ -76,6 +76,30 @@ public class ElasticsearchCliTests extends ESTestCase { runTest(expectedStatus, false, outputConsumer, (foreground, pidFile, esSettings) -> {}, args); } + public void testPositionalArgs() throws Exception { + runTest( + ExitCodes.USAGE, + false, + output -> assertThat(output, containsString("Positional arguments not allowed, found [foo]")), + (foreground, pidFile, esSettings) -> {}, + "foo" + ); + runTest( + ExitCodes.USAGE, + false, + output -> assertThat(output, containsString("Positional arguments not allowed, found [foo, bar]")), + (foreground, pidFile, esSettings) -> {}, + "foo", "bar" + ); + runTest( + ExitCodes.USAGE, + false, + output -> assertThat(output, containsString("Positional arguments not allowed, found [foo]")), + (foreground, pidFile, esSettings) -> {}, + "-E", "something", "foo", "-E", "somethingelse" + ); + } + public void testThatPidFileCanBeConfigured() throws Exception { runPidFileTest(ExitCodes.USAGE, false, output -> assertThat(output, containsString("Option p/pidfile requires an argument")), "-p"); runPidFileTest(ExitCodes.OK, true, output -> {}, "-p", "/tmp/pid"); diff --git a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java index 95a65f82924..7b34685da7d 100644 --- a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java +++ b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java @@ -154,7 +154,7 @@ public class PagedBytesReferenceTests extends ESTestCase { } public void testStreamInputBulkReadWithOffset() throws IOException { - int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20)); + final int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20)); BytesReference pbr = getRandomizedPagedBytesReference(length); StreamInput si = pbr.streamInput(); assertNotNull(si); @@ -162,6 +162,7 @@ public class PagedBytesReferenceTests extends ESTestCase { // read a bunch of single bytes one by one int offset = randomIntBetween(1, length / 2); for (int i = 0; i < offset; i++) { + assertEquals(si.available(), length - i); assertEquals(pbr.get(i), si.readByte()); } @@ -176,6 +177,7 @@ public class PagedBytesReferenceTests extends ESTestCase { // bulk-read all si.readFully(targetBytes); assertArrayEquals(pbrBytesWithOffset, targetBytes); + assertEquals(si.available(), 0); } public void testRandomReads() throws IOException { @@ -216,18 +218,22 @@ public class PagedBytesReferenceTests extends ESTestCase { int sliceLength = length - sliceOffset; BytesReference slice = pbr.slice(sliceOffset, sliceLength); StreamInput sliceInput = slice.streamInput(); + assertEquals(sliceInput.available(), sliceLength); // single reads assertEquals(slice.get(0), sliceInput.readByte()); assertEquals(slice.get(1), sliceInput.readByte()); assertEquals(slice.get(2), sliceInput.readByte()); + assertEquals(sliceInput.available(), sliceLength - 3); // reset the slice stream for bulk reading sliceInput.reset(); + assertEquals(sliceInput.available(), sliceLength); // bulk read byte[] sliceBytes = new byte[sliceLength]; sliceInput.readFully(sliceBytes); + assertEquals(sliceInput.available(), 0); // compare slice content with upper half of original byte[] pbrSliceBytes = Arrays.copyOfRange(pbr.toBytes(), sliceOffset, length); @@ -239,11 +245,14 @@ public class PagedBytesReferenceTests extends ESTestCase { assertArrayEquals(sliceBytes, sliceToBytes); sliceInput.reset(); + assertEquals(sliceInput.available(), sliceLength); byte[] buffer = new byte[sliceLength + scaledRandomIntBetween(1, 100)]; int offset = scaledRandomIntBetween(0, Math.max(1, buffer.length - sliceLength - 1)); int read = sliceInput.read(buffer, offset, sliceLength / 2); + assertEquals(sliceInput.available(), sliceLength - read); sliceInput.read(buffer, offset + read, sliceLength); assertArrayEquals(sliceBytes, Arrays.copyOfRange(buffer, offset, offset + sliceLength)); + assertEquals(sliceInput.available(), 0); } public void testWriteToOutputStream() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java index 7f232363f73..80bad2e1ecc 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java @@ -273,7 +273,9 @@ public class BytesStreamsTests extends ESTestCase { out.writeString("hello"); out.writeString("goodbye"); out.writeGenericValue(BytesRefs.toBytesRef("bytesref")); + final byte[] bytes = out.bytes().toBytes(); StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + assertEquals(in.available(), bytes.length); assertThat(in.readBoolean(), equalTo(false)); assertThat(in.readByte(), equalTo((byte)1)); assertThat(in.readShort(), equalTo((short)-1)); @@ -302,9 +304,12 @@ public class BytesStreamsTests extends ESTestCase { namedWriteableRegistry.registerPrototype(BaseNamedWriteable.class, new TestNamedWriteable(null, null)); TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); - StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes().toBytes()), namedWriteableRegistry); + byte[] bytes = out.bytes().toBytes(); + StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); + assertEquals(in.available(), bytes.length); BaseNamedWriteable namedWriteableOut = in.readNamedWriteable(BaseNamedWriteable.class); assertEquals(namedWriteableOut, namedWriteableIn); + assertEquals(in.available(), 0); } public void testNamedWriteableDuplicates() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java index b5f26dba8a5..72f933462e0 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java @@ -23,7 +23,10 @@ import org.elasticsearch.common.bytes.ByteBufferBytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.test.ESTestCase; +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -93,4 +96,29 @@ public class StreamTests extends ESTestCase { index++; } } + + public void testFilterStreamInputDelegatesAvailable() throws IOException { + final int length = randomIntBetween(1, 1024); + StreamInput delegate = StreamInput.wrap(new byte[length]); + + FilterStreamInput filterInputStream = new FilterStreamInput(delegate) {}; + assertEquals(filterInputStream.available(), length); + + // read some bytes + final int bytesToRead = randomIntBetween(1, length); + filterInputStream.readBytes(new byte[bytesToRead], 0, bytesToRead); + assertEquals(filterInputStream.available(), length - bytesToRead); + } + + public void testInputStreamStreamInputDelegatesAvailable() throws IOException { + final int length = randomIntBetween(1, 1024); + ByteArrayInputStream is = new ByteArrayInputStream(new byte[length]); + InputStreamStreamInput streamInput = new InputStreamStreamInput(is); + assertEquals(streamInput.available(), length); + + // read some bytes + final int bytesToRead = randomIntBetween(1, length); + streamInput.readBytes(new byte[bytesToRead], 0, bytesToRead); + assertEquals(streamInput.available(), length - bytesToRead); + } } diff --git a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java b/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java index 4d8fbc3add5..405ac566686 100644 --- a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java +++ b/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.test.ESTestCase; import java.util.LinkedHashMap; import java.util.Map; +import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.is; public class PropertyPlaceholderTests extends ESTestCase { @@ -33,10 +34,10 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("foo1", "bar1"); map.put("foo2", "bar2"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("bar1", propertyPlaceholder.replacePlaceholders("{foo1}", placeholderResolver)); - assertEquals("a bar1b", propertyPlaceholder.replacePlaceholders("a {foo1}b", placeholderResolver)); - assertEquals("bar1bar2", propertyPlaceholder.replacePlaceholders("{foo1}{foo2}", placeholderResolver)); - assertEquals("a bar1 b bar2 c", propertyPlaceholder.replacePlaceholders("a {foo1} b {foo2} c", placeholderResolver)); + assertEquals("bar1", propertyPlaceholder.replacePlaceholders("key", "{foo1}", placeholderResolver)); + assertEquals("a bar1b", propertyPlaceholder.replacePlaceholders("key", "a {foo1}b", placeholderResolver)); + assertEquals("bar1bar2", propertyPlaceholder.replacePlaceholders("key", "{foo1}{foo2}", placeholderResolver)); + assertEquals("a bar1 b bar2 c", propertyPlaceholder.replacePlaceholders("key", "a {foo1} b {foo2} c", placeholderResolver)); } public void testVariousPrefixSuffix() { @@ -47,24 +48,24 @@ public class PropertyPlaceholderTests extends ESTestCase { Map map = new LinkedHashMap<>(); map.put("foo", "bar"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("bar", ppEqualsPrefix.replacePlaceholders("{foo}", placeholderResolver)); - assertEquals("bar", ppLongerPrefix.replacePlaceholders("${foo}", placeholderResolver)); - assertEquals("bar", ppShorterPrefix.replacePlaceholders("{foo}}", placeholderResolver)); + assertEquals("bar", ppEqualsPrefix.replacePlaceholders("key", "{foo}", placeholderResolver)); + assertEquals("bar", ppLongerPrefix.replacePlaceholders("key", "${foo}", placeholderResolver)); + assertEquals("bar", ppShorterPrefix.replacePlaceholders("key", "{foo}}", placeholderResolver)); } public void testDefaultValue() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("bar", propertyPlaceholder.replacePlaceholders("${foo:bar}", placeholderResolver)); - assertEquals("", propertyPlaceholder.replacePlaceholders("${foo:}", placeholderResolver)); + assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "${foo:bar}", placeholderResolver)); + assertEquals("", propertyPlaceholder.replacePlaceholders("key", "${foo:}", placeholderResolver)); } public void testIgnoredUnresolvedPlaceholder() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", true); Map map = new LinkedHashMap<>(); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("${foo}", propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver)); + assertEquals("${foo}", propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver)); } public void testNotIgnoredUnresolvedPlaceholder() { @@ -72,7 +73,7 @@ public class PropertyPlaceholderTests extends ESTestCase { Map map = new LinkedHashMap<>(); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); try { - propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver); + propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), is("Could not resolve placeholder 'foo'")); @@ -83,7 +84,7 @@ public class PropertyPlaceholderTests extends ESTestCase { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, true); - assertEquals("bar", propertyPlaceholder.replacePlaceholders("bar${foo}", placeholderResolver)); + assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "bar${foo}", placeholderResolver)); } public void testRecursive() { @@ -93,8 +94,8 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("foo1", "${foo2}"); map.put("foo2", "bar"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("bar", propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver)); - assertEquals("abarb", propertyPlaceholder.replacePlaceholders("a${foo}b", placeholderResolver)); + assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver)); + assertEquals("abarb", propertyPlaceholder.replacePlaceholders("key", "a${foo}b", placeholderResolver)); } public void testNestedLongerPrefix() { @@ -105,7 +106,7 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("foo2", "bar"); map.put("barbar", "baz"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("baz", propertyPlaceholder.replacePlaceholders("${bar${foo}}", placeholderResolver)); + assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "${bar${foo}}", placeholderResolver)); } public void testNestedSameLengthPrefixSuffix() { @@ -116,7 +117,7 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("foo2", "bar"); map.put("barbar", "baz"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("baz", propertyPlaceholder.replacePlaceholders("{bar{foo}}", placeholderResolver)); + assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "{bar{foo}}", placeholderResolver)); } public void testNestedShorterPrefix() { @@ -127,7 +128,7 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("foo2", "bar"); map.put("barbar", "baz"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); - assertEquals("baz", propertyPlaceholder.replacePlaceholders("{bar{foo}}}}", placeholderResolver)); + assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "{bar{foo}}}}", placeholderResolver)); } public void testCircularReference() { @@ -137,7 +138,7 @@ public class PropertyPlaceholderTests extends ESTestCase { map.put("bar", "${foo}"); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true); try { - propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver); + propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), is("Circular placeholder reference 'foo' in property definitions")); @@ -148,7 +149,24 @@ public class PropertyPlaceholderTests extends ESTestCase { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); Map map = new LinkedHashMap<>(); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false); - assertEquals("bar${foo}", propertyPlaceholder.replacePlaceholders("bar${foo}", placeholderResolver)); + assertEquals("bar${foo}", propertyPlaceholder.replacePlaceholders("key", "bar${foo}", placeholderResolver)); + } + + public void testNullKey() { + final PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); + final Map map = new LinkedHashMap<>(); + final PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false); + expectThrows(NullPointerException.class, () -> propertyPlaceholder.replacePlaceholders(null, "value", placeholderResolver)); + } + + public void testNullValue() { + final PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); + final Map map = new LinkedHashMap<>(); + final PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false); + final String key = randomAsciiOfLength(10); + NullPointerException e = + expectThrows(NullPointerException.class, () -> propertyPlaceholder.replacePlaceholders(key, null, placeholderResolver)); + assertThat(e, hasToString("java.lang.NullPointerException: value can not be null for [" + key + "]")); } private class SimplePlaceholderResolver implements PropertyPlaceholder.PlaceholderResolver { diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java index b7ce4c305e1..d14bd583c44 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java @@ -275,7 +275,7 @@ public class ZenDiscoveryIT extends ESIntegTestCase { Settings nodeSettings = Settings.settingsBuilder() .put("discovery.type", "zen") // <-- To override the local setting if set externally .build(); - String nodeName = internalCluster().startNode(nodeSettings, Version.V_5_0_0); + String nodeName = internalCluster().startNode(nodeSettings, Version.V_5_0_0_alpha1); ZenDiscovery zenDiscovery = (ZenDiscovery) internalCluster().getInstance(Discovery.class, nodeName); ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName); DiscoveryNode node = new DiscoveryNode("_node_id", new InetSocketTransportAddress(InetAddress.getByName("0.0.0.0"), 0), Version.V_2_0_0); @@ -292,13 +292,13 @@ public class ZenDiscoveryIT extends ESIntegTestCase { }); assertThat(holder.get(), notNullValue()); - assertThat(holder.get().getMessage(), equalTo("Can't handle join request from a node with a version [2.0.0] that is lower than the minimum compatible version [" + Version.V_5_0_0.minimumCompatibilityVersion() + "]")); + assertThat(holder.get().getMessage(), equalTo("Can't handle join request from a node with a version [2.0.0] that is lower than the minimum compatible version [" + Version.V_5_0_0_alpha1.minimumCompatibilityVersion() + "]")); } public void testJoinElectedMaster_incompatibleMinVersion() { - ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY, Version.V_5_0_0); + ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY, Version.V_5_0_0_alpha1); - DiscoveryNode node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.V_5_0_0); + DiscoveryNode node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.V_5_0_0_alpha1); assertThat(electMasterService.electMaster(Collections.singletonList(node)), sameInstance(node)); node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.V_2_0_0); assertThat("Can't join master because version 2.0.0 is lower than the minimum compatable version 5.0.0 can support", electMasterService.electMaster(Collections.singletonList(node)), nullValue()); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java index 1fd9655e8de..f665de6216d 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java @@ -76,7 +76,7 @@ public class AnalysisServiceTests extends ESTestCase { } public void testOverrideDefaultIndexAnalyzer() { - Version version = VersionUtils.randomVersionBetween(getRandom(), Version.V_5_0_0, Version.CURRENT); + Version version = VersionUtils.randomVersionBetween(getRandom(), Version.V_5_0_0_alpha1, Version.CURRENT); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); try { AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings), @@ -90,7 +90,7 @@ public class AnalysisServiceTests extends ESTestCase { } public void testBackCompatOverrideDefaultIndexAnalyzer() { - Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0)); + Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1)); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings), Collections.singletonMap("default_index", analyzerProvider("default_index")), @@ -112,7 +112,7 @@ public class AnalysisServiceTests extends ESTestCase { } public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() { - Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0)); + Version version = VersionUtils.randomVersionBetween(getRandom(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1)); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); Map analyzers = new HashMap<>(); analyzers.put("default_index", analyzerProvider("default_index")); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java index 8ee8332b708..31b65e84aa0 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java @@ -39,7 +39,7 @@ public class DynamicTemplateTests extends ESTestCase { templateDef.put("random_param", "random_value"); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0)); + () -> DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1)); assertEquals("Illegal dynamic template parameter: [random_param]", e.getMessage()); // but no issues on 2.x for bw compat @@ -54,7 +54,7 @@ public class DynamicTemplateTests extends ESTestCase { Map templateDef = new HashMap<>(); templateDef.put("match_mapping_type", "string"); templateDef.put("mapping", Collections.singletonMap("store", true)); - DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0); + DynamicTemplate template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1); XContentBuilder builder = JsonXContent.contentBuilder(); template.toXContent(builder, ToXContent.EMPTY_PARAMS); assertEquals("{\"match_mapping_type\":\"string\",\"mapping\":{\"store\":true}}", builder.string()); @@ -64,7 +64,7 @@ public class DynamicTemplateTests extends ESTestCase { templateDef.put("match", "*name"); templateDef.put("unmatch", "first_name"); templateDef.put("mapping", Collections.singletonMap("store", true)); - template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0); + template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1); builder = JsonXContent.contentBuilder(); template.toXContent(builder, ToXContent.EMPTY_PARAMS); assertEquals("{\"match\":\"*name\",\"unmatch\":\"first_name\",\"mapping\":{\"store\":true}}", builder.string()); @@ -74,7 +74,7 @@ public class DynamicTemplateTests extends ESTestCase { templateDef.put("path_match", "*name"); templateDef.put("path_unmatch", "first_name"); templateDef.put("mapping", Collections.singletonMap("store", true)); - template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0); + template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1); builder = JsonXContent.contentBuilder(); template.toXContent(builder, ToXContent.EMPTY_PARAMS); assertEquals("{\"path_match\":\"*name\",\"path_unmatch\":\"first_name\",\"mapping\":{\"store\":true}}", @@ -85,7 +85,7 @@ public class DynamicTemplateTests extends ESTestCase { templateDef.put("match", "^a$"); templateDef.put("match_pattern", "regex"); templateDef.put("mapping", Collections.singletonMap("store", true)); - template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0); + template = DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha1); builder = JsonXContent.contentBuilder(); template.toXContent(builder, ToXContent.EMPTY_PARAMS); assertEquals("{\"match\":\"^a$\",\"match_pattern\":\"regex\",\"mapping\":{\"store\":true}}", builder.string()); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java index f98c74a7d25..92cee7c5407 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java @@ -126,7 +126,7 @@ public class StringMappingUpgradeTests extends ESSingleNodeTestCase { .field("type", "string") .field("index", keyword ? "not_analyzed" : "analyzed") .startObject("fielddata") - .field("forwat", format) + .field("format", format) .field("loading", loading) .startObject("filter") .startObject("frequency") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java index 8c25713ce3d..e1b94967b43 100755 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java @@ -125,16 +125,6 @@ public class ExternalMapper extends FieldMapper { public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { ExternalMapper.Builder builder = new ExternalMapper.Builder(name, generatedValue, mapperName); parseField(builder, name, node, parserContext); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String propName = Strings.toUnderscoreCase(entry.getKey()); - Object propNode = entry.getValue(); - - if (parseMultiField(builder, name, parserContext, propName, propNode)) { - iterator.remove(); - } - } - return builder; } } diff --git a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java b/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java index f23ec6d9595..f17a4fc664d 100644 --- a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -102,6 +103,27 @@ public class ExtractQueryTermsServiceTests extends ESTestCase { assertThat(terms.get(0).bytes(), equalTo(termQuery.getTerm().bytes())); } + public void testExtractQueryMetadata_termsQuery() { + TermsQuery termsQuery = new TermsQuery("_field", new BytesRef("_term1"), new BytesRef("_term2")); + List terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(termsQuery)); + Collections.sort(terms); + assertThat(terms.size(), equalTo(2)); + assertThat(terms.get(0).field(), equalTo("_field")); + assertThat(terms.get(0).text(), equalTo("_term1")); + assertThat(terms.get(1).field(), equalTo("_field")); + assertThat(terms.get(1).text(), equalTo("_term2")); + + // test with different fields + termsQuery = new TermsQuery(new Term("_field1", "_term1"), new Term("_field2", "_term2")); + terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(termsQuery)); + Collections.sort(terms); + assertThat(terms.size(), equalTo(2)); + assertThat(terms.get(0).field(), equalTo("_field1")); + assertThat(terms.get(0).text(), equalTo("_term1")); + assertThat(terms.get(1).field(), equalTo("_field2")); + assertThat(terms.get(1).text(), equalTo("_term2")); + } + public void testExtractQueryMetadata_phraseQuery() { PhraseQuery phraseQuery = new PhraseQuery("_field", "_term1", "term2"); List terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(phraseQuery)); diff --git a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java index 8711e89883f..62dec09e906 100644 --- a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java +++ b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java @@ -22,13 +22,15 @@ package org.elasticsearch.index.suggest.stats; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.suggest.SuggestRequestBuilder; -import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.search.stats.SearchStats; +import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; import org.elasticsearch.test.ESIntegTestCase; @@ -86,49 +88,50 @@ public class SuggestStatsIT extends ESIntegTestCase { long startTime = System.currentTimeMillis(); for (int i = 0; i < suggestAllIdx; i++) { - SuggestResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSuggest(), i).get(); + SearchResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSearch(), i).get(); assertAllSuccessful(suggestResponse); } for (int i = 0; i < suggestIdx1; i++) { - SuggestResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSuggest("test1"), i).get(); + SearchResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSearch("test1"), i).get(); assertAllSuccessful(suggestResponse); } for (int i = 0; i < suggestIdx2; i++) { - SuggestResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSuggest("test2"), i).get(); + SearchResponse suggestResponse = addSuggestions(internalCluster().clientNodeClient().prepareSearch("test2"), i).get(); assertAllSuccessful(suggestResponse); } long endTime = System.currentTimeMillis(); IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().execute().actionGet(); + final SearchStats.Stats suggest = indicesStats.getTotal().getSearch().getTotal(); // check current - assertThat(indicesStats.getTotal().getSuggest().getCurrent(), equalTo(0L)); + assertThat(suggest.getSuggestCurrent(), equalTo(0L)); // check suggest count - assertThat(indicesStats.getTotal().getSuggest().getCount(), equalTo((long) (suggestAllIdx * totalShards + suggestIdx1 * shardsIdx1 + suggestIdx2 * shardsIdx2))); - assertThat(indicesStats.getIndices().get("test1").getTotal().getSuggest().getCount(), equalTo((long) ((suggestAllIdx + suggestIdx1) * shardsIdx1))); - assertThat(indicesStats.getIndices().get("test2").getTotal().getSuggest().getCount(), equalTo((long) ((suggestAllIdx + suggestIdx2) * shardsIdx2))); + assertThat(suggest.getSuggestCount(), equalTo((long) (suggestAllIdx * totalShards + suggestIdx1 * shardsIdx1 + suggestIdx2 * shardsIdx2))); + assertThat(indicesStats.getIndices().get("test1").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx1) * shardsIdx1))); + assertThat(indicesStats.getIndices().get("test2").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx2) * shardsIdx2))); logger.info("iter {}, iter1 {}, iter2 {}, {}", suggestAllIdx, suggestIdx1, suggestIdx2, endTime - startTime); // check suggest time - assertThat(indicesStats.getTotal().getSuggest().getTimeInMillis(), greaterThan(0L)); + assertThat(suggest.getSuggestTimeInMillis(), greaterThan(0L)); // the upperbound is num shards * total time since we do searches in parallel - assertThat(indicesStats.getTotal().getSuggest().getTimeInMillis(), lessThanOrEqualTo(totalShards * (endTime - startTime))); + assertThat(suggest.getSuggestTimeInMillis(), lessThanOrEqualTo(totalShards * (endTime - startTime))); NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet(); NodeStats[] nodes = nodeStats.getNodes(); Set nodeIdsWithIndex = nodeIdsWithIndex("test1", "test2"); int num = 0; for (NodeStats stat : nodes) { - SuggestStats suggestStats = stat.getIndices().getSuggest(); + SearchStats.Stats suggestStats = stat.getIndices().getSearch().getTotal(); logger.info("evaluating {}", stat.getNode()); if (nodeIdsWithIndex.contains(stat.getNode().getId())) { - assertThat(suggestStats.getCount(), greaterThan(0L)); - assertThat(suggestStats.getTimeInMillis(), greaterThan(0L)); + assertThat(suggestStats.getSuggestCount(), greaterThan(0L)); + assertThat(suggestStats.getSuggestTimeInMillis(), greaterThan(0L)); num++; } else { - assertThat(suggestStats.getCount(), equalTo(0L)); - assertThat(suggestStats.getTimeInMillis(), equalTo(0L)); + assertThat(suggestStats.getSuggestCount(), equalTo(0L)); + assertThat(suggestStats.getSuggestTimeInMillis(), equalTo(0L)); } } @@ -136,15 +139,16 @@ public class SuggestStatsIT extends ESIntegTestCase { } - private SuggestRequestBuilder addSuggestions(SuggestRequestBuilder request, int i) { + private SearchRequestBuilder addSuggestions(SearchRequestBuilder request, int i) { + final SuggestBuilder suggestBuilder = new SuggestBuilder(); for (int s = 0; s < randomIntBetween(2, 10); s++) { if (randomBoolean()) { - request.addSuggestion("s" + s, new PhraseSuggestionBuilder("f").text("test" + i + " test" + (i - 1))); + suggestBuilder.addSuggestion("s" + s, new PhraseSuggestionBuilder("f").text("test" + i + " test" + (i - 1))); } else { - request.addSuggestion("s" + s, new TermSuggestionBuilder("f").text("test" + i)); + suggestBuilder.addSuggestion("s" + s, new TermSuggestionBuilder("f").text("test" + i)); } } - return request; + return request.suggest(suggestBuilder); } private Set nodeIdsWithIndex(String... indices) { diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java index 8ea053a64ab..046d0b4400f 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java @@ -44,7 +44,6 @@ import org.elasticsearch.action.search.MultiSearchRequestBuilder; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.suggest.SuggestRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; @@ -53,6 +52,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.SuggestBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -94,7 +94,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1", "test2"), true); verify(percolate("test1", "test2"), true); verify(mpercolate(null, "test1", "test2"), false); - verify(suggest("test1", "test2"), true); verify(getAliases("test1", "test2"), true); verify(getFieldMapping("test1", "test2"), true); verify(getMapping("test1", "test2"), true); @@ -114,7 +113,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1", "test2").setIndicesOptions(options), true); verify(percolate("test1", "test2").setIndicesOptions(options), true); verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false); - verify(suggest("test1", "test2").setIndicesOptions(options), true); verify(getAliases("test1", "test2").setIndicesOptions(options), true); verify(getFieldMapping("test1", "test2").setIndicesOptions(options), true); verify(getMapping("test1", "test2").setIndicesOptions(options), true); @@ -134,7 +132,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1", "test2").setIndicesOptions(options), false); verify(percolate("test1", "test2").setIndicesOptions(options), false); verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false); - verify(suggest("test1", "test2").setIndicesOptions(options), false); verify(getAliases("test1", "test2").setIndicesOptions(options), false); verify(getFieldMapping("test1", "test2").setIndicesOptions(options), false); verify(getMapping("test1", "test2").setIndicesOptions(options), false); @@ -156,7 +153,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1", "test2").setIndicesOptions(options), false); verify(percolate("test1", "test2").setIndicesOptions(options), false); verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false); - verify(suggest("test1", "test2").setIndicesOptions(options), false); verify(getAliases("test1", "test2").setIndicesOptions(options), false); verify(getFieldMapping("test1", "test2").setIndicesOptions(options), false); verify(getMapping("test1", "test2").setIndicesOptions(options), false); @@ -186,7 +182,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1").setIndicesOptions(options), true); verify(percolate("test1").setIndicesOptions(options), true); verify(mpercolate(options, "test1").setIndicesOptions(options), true); - verify(suggest("test1").setIndicesOptions(options), true); verify(getAliases("test1").setIndicesOptions(options), true); verify(getFieldMapping("test1").setIndicesOptions(options), true); verify(getMapping("test1").setIndicesOptions(options), true); @@ -206,7 +201,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1").setIndicesOptions(options), false); verify(percolate("test1").setIndicesOptions(options), false); verify(mpercolate(options, "test1").setIndicesOptions(options), false); - verify(suggest("test1").setIndicesOptions(options), false); verify(getAliases("test1").setIndicesOptions(options), false); verify(getFieldMapping("test1").setIndicesOptions(options), false); verify(getMapping("test1").setIndicesOptions(options), false); @@ -229,7 +223,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists("test1").setIndicesOptions(options), false); verify(percolate("test1").setIndicesOptions(options), false); verify(mpercolate(options, "test1").setIndicesOptions(options), false); - verify(suggest("test1").setIndicesOptions(options), false); verify(getAliases("test1").setIndicesOptions(options), false); verify(getFieldMapping("test1").setIndicesOptions(options), false); verify(getMapping("test1").setIndicesOptions(options), false); @@ -250,7 +243,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(aliasExists("test1").setIndicesOptions(options), true); verify(typesExists("test1").setIndicesOptions(options), true); verify(percolate("test1").setIndicesOptions(options), true); - verify(suggest("test1").setIndicesOptions(options), true); verify(getAliases("test1").setIndicesOptions(options), true); verify(getFieldMapping("test1").setIndicesOptions(options), true); verify(getMapping("test1").setIndicesOptions(options), true); @@ -269,7 +261,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(aliasExists("test1").setIndicesOptions(options), false); verify(typesExists("test1").setIndicesOptions(options), false); verify(percolate("test1").setIndicesOptions(options), false); - verify(suggest("test1").setIndicesOptions(options), false); verify(getAliases("test1").setIndicesOptions(options), false); verify(getFieldMapping("test1").setIndicesOptions(options), false); verify(getMapping("test1").setIndicesOptions(options), false); @@ -291,7 +282,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(aliasExists("test1").setIndicesOptions(options), false); verify(typesExists("test1").setIndicesOptions(options), false); verify(percolate("test1").setIndicesOptions(options), false); - verify(suggest("test1").setIndicesOptions(options), false); verify(getAliases("test1").setIndicesOptions(options), false); verify(getFieldMapping("test1").setIndicesOptions(options), false); verify(getMapping("test1").setIndicesOptions(options), false); @@ -344,7 +334,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists(indices), false); verify(percolate(indices), false); verify(mpercolate(null, indices), false); - verify(suggest(indices), false); verify(getAliases(indices), false); verify(getFieldMapping(indices), false); verify(getMapping(indices), false); @@ -365,7 +354,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists(indices).setIndicesOptions(options), false); verify(percolate(indices).setIndicesOptions(options), false); verify(mpercolate(options, indices), false); - verify(suggest(indices).setIndicesOptions(options), false); verify(getAliases(indices).setIndicesOptions(options), false); verify(getFieldMapping(indices).setIndicesOptions(options), false); verify(getMapping(indices).setIndicesOptions(options), false); @@ -389,7 +377,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists(indices), false); verify(percolate(indices), false); verify(mpercolate(null, indices), false); - verify(suggest(indices), false); verify(getAliases(indices), false); verify(getFieldMapping(indices), false); verify(getMapping(indices), false); @@ -410,7 +397,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists(indices), false); verify(percolate(indices), false); verify(mpercolate(null, indices), false); - verify(suggest(indices), false); verify(getAliases(indices), false); verify(getFieldMapping(indices), false); verify(getMapping(indices), false); @@ -431,7 +417,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { verify(typesExists(indices).setIndicesOptions(options), false); verify(percolate(indices).setIndicesOptions(options), false); verify(mpercolate(options, indices), false); - verify(suggest(indices).setIndicesOptions(options), false); verify(getAliases(indices).setIndicesOptions(options), false); verify(getFieldMapping(indices).setIndicesOptions(options), false); verify(getMapping(indices).setIndicesOptions(options), false); @@ -755,10 +740,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase { return builder.add(percolate(indices)); } - private static SuggestRequestBuilder suggest(String... indices) { - return client().prepareSuggest(indices).addSuggestion("name", SuggestBuilders.termSuggestion("a")); - } - private static GetAliasesRequestBuilder getAliases(String... indices) { return client().admin().indices().prepareGetAliases("dummy").addIndices(indices); } diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index b7cb64a7d2f..12e6d2b4799 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -61,6 +61,7 @@ import java.util.Random; import static org.elasticsearch.cluster.metadata.IndexMetaData.PROTO; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.common.xcontent.XContentFactory.contentBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; @@ -575,6 +576,10 @@ public class IndexStatsIT extends ESIntegTestCase { IndicesStatsResponse stats = builder.execute().actionGet(); for (Flag flag : values) { + if (flag == Flag.Suggest) { + // suggest flag is unused + continue; + } assertThat(isSet(flag, stats.getPrimaries()), equalTo(false)); assertThat(isSet(flag, stats.getTotal()), equalTo(false)); } @@ -610,6 +615,10 @@ public class IndexStatsIT extends ESIntegTestCase { } for (Flag flag : EnumSet.complementOf(flags)) { // check the complement + if (flag == Flag.Suggest) { + // suggest flag is unused + continue; + } assertThat(isSet(flag, stats.getPrimaries()), equalTo(false)); assertThat(isSet(flag, stats.getTotal()), equalTo(false)); } @@ -914,8 +923,7 @@ public class IndexStatsIT extends ESIntegTestCase { case Translog: builder.setTranslog(set); break; - case Suggest: - builder.setSuggest(set); + case Suggest: // unused break; case RequestCache: builder.setRequestCache(set); @@ -961,8 +969,8 @@ public class IndexStatsIT extends ESIntegTestCase { return response.getSegments() != null; case Translog: return response.getTranslog() != null; - case Suggest: - return response.getSuggest() != null; + case Suggest: // unused + return true; case RequestCache: return response.getRequestCache() != null; case Recovery: diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java index 04bff31057d..73b31b92637 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginInfoTests.java @@ -46,7 +46,6 @@ public class PluginInfoTests extends ESTestCase { assertEquals("fake desc", info.getDescription()); assertEquals("1.0", info.getVersion()); assertEquals("FakePlugin", info.getClassname()); - assertTrue(info.isIsolated()); } public void testReadFromPropertiesNameMissing() throws Exception { @@ -203,11 +202,11 @@ public class PluginInfoTests extends ESTestCase { public void testPluginListSorted() { PluginsAndModules pluginsInfo = new PluginsAndModules(); - pluginsInfo.addPlugin(new PluginInfo("c", "foo", "dummy", "dummyclass", true)); - pluginsInfo.addPlugin(new PluginInfo("b", "foo", "dummy", "dummyclass", true)); - pluginsInfo.addPlugin(new PluginInfo("e", "foo", "dummy", "dummyclass", true)); - pluginsInfo.addPlugin(new PluginInfo("a", "foo", "dummy", "dummyclass", true)); - pluginsInfo.addPlugin(new PluginInfo("d", "foo", "dummy", "dummyclass", true)); + pluginsInfo.addPlugin(new PluginInfo("c", "foo", "dummy", "dummyclass")); + pluginsInfo.addPlugin(new PluginInfo("b", "foo", "dummy", "dummyclass")); + pluginsInfo.addPlugin(new PluginInfo("e", "foo", "dummy", "dummyclass")); + pluginsInfo.addPlugin(new PluginInfo("a", "foo", "dummy", "dummyclass")); + pluginsInfo.addPlugin(new PluginInfo("d", "foo", "dummy", "dummyclass")); final List infos = pluginsInfo.getPluginInfos(); List names = infos.stream().map((input) -> input.getName()).collect(Collectors.toList()); diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index df992a06aae..74b7b17a037 100644 --- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -74,6 +74,8 @@ import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilderTests; import org.elasticsearch.search.rescore.QueryRescoreBuilderTests; import org.elasticsearch.search.searchafter.SearchAfterBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -551,7 +553,7 @@ public class SearchSourceBuilderTests extends ESTestCase { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser), aggParsers, suggesters); assertEquals(1, searchSourceBuilder.sorts().size()); - assertEquals("{\"foo\":{\"order\":\"asc\"}}", searchSourceBuilder.sorts().get(0).toUtf8()); + assertEquals(new FieldSortBuilder("foo"), searchSourceBuilder.sorts().get(0)); } } @@ -567,11 +569,11 @@ public class SearchSourceBuilderTests extends ESTestCase { SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.parseSearchSource(parser, createParseContext(parser), aggParsers, suggesters); assertEquals(5, searchSourceBuilder.sorts().size()); - assertEquals("{\"post_date\":{\"order\":\"asc\"}}", searchSourceBuilder.sorts().get(0).toUtf8()); - assertEquals("\"user\"", searchSourceBuilder.sorts().get(1).toUtf8()); - assertEquals("{\"name\":\"desc\"}", searchSourceBuilder.sorts().get(2).toUtf8()); - assertEquals("{\"age\":\"desc\"}", searchSourceBuilder.sorts().get(3).toUtf8()); - assertEquals("\"_score\"", searchSourceBuilder.sorts().get(4).toUtf8()); + assertEquals(new FieldSortBuilder("post_date"), searchSourceBuilder.sorts().get(0)); + assertEquals(new FieldSortBuilder("user"), searchSourceBuilder.sorts().get(1)); + assertEquals(new FieldSortBuilder("name").order(SortOrder.DESC), searchSourceBuilder.sorts().get(2)); + assertEquals(new FieldSortBuilder("age").order(SortOrder.DESC), searchSourceBuilder.sorts().get(3)); + assertEquals(new ScoreSortBuilder(), searchSourceBuilder.sorts().get(4)); } } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index c3e31bf2899..935dfb178b5 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -66,7 +66,6 @@ import org.junit.BeforeClass; import java.io.IOException; import java.nio.file.Path; import java.util.Collections; -import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.equalTo; @@ -78,7 +77,6 @@ public abstract class AbstractSortTestCase> extends EST private static final int NUMBER_OF_TESTBUILDERS = 20; static IndicesQueriesRegistry indicesQueriesRegistry; - private static SortParseElement parseElement = new SortParseElement(); private static ScriptService scriptService; @BeforeClass @@ -131,10 +129,7 @@ public abstract class AbstractSortTestCase> extends EST if (randomBoolean()) { builder.prettyPrint(); } - builder.startObject(); testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - XContentParser itemParser = XContentHelper.createParser(builder.bytes()); itemParser.nextToken(); @@ -163,24 +158,12 @@ public abstract class AbstractSortTestCase> extends EST for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { T sortBuilder = createTestItem(); SortField sortField = sortBuilder.build(mockShardContext); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); - if (randomBoolean()) { - builder.prettyPrint(); - } - builder.startObject(); - sortBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - XContentParser parser = XContentHelper.createParser(builder.bytes()); - parser.nextToken(); - List sortFields = parseElement.parse(parser, mockShardContext); - assertEquals(1, sortFields.size()); - SortField sortFieldOldStyle = sortFields.get(0); - assertEquals(sortFieldOldStyle.getField(), sortField.getField()); - assertEquals(sortFieldOldStyle.getReverse(), sortField.getReverse()); - assertEquals(sortFieldOldStyle.getType(), sortField.getType()); + sortFieldAssertions(sortBuilder, sortField); } } + protected abstract void sortFieldAssertions(T builder, SortField sortField) throws IOException; + /** * Test serialization and deserialization of the test sort. */ diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index 74b56353a91..2c1a4a65c7b 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -19,6 +19,8 @@ x * Licensed to Elasticsearch under one or more contributor package org.elasticsearch.search.sort; +import org.apache.lucene.search.SortField; + import java.io.IOException; public class FieldSortBuilderTests extends AbstractSortTestCase { @@ -29,7 +31,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase 1) { @@ -195,7 +196,7 @@ public class SortBuilderTests extends ESTestCase { list.add(new ScoreSortBuilder()); break; case 1: - String fieldName = rarely() ? SortParseElement.DOC_FIELD_NAME : randomAsciiOfLengthBetween(1, 10); + String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAsciiOfLengthBetween(1, 10); list.add(new FieldSortBuilder(fieldName)); break; case 2: diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java index 0c64b7e7b15..23d12fcfd0b 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java @@ -19,12 +19,14 @@ package org.elasticsearch.search.sort; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.TestSearchContext; @@ -47,10 +49,7 @@ public class SortParserTests extends ESSingleNodeTestCase { sortBuilder.field("unit", "km"); sortBuilder.field("sort_mode", "max"); sortBuilder.endObject(); - XContentParser parser = XContentHelper.createParser(sortBuilder.bytes()); - parser.nextToken(); - GeoDistanceSortParser geoParser = new GeoDistanceSortParser(); - geoParser.parse(parser, context.getQueryShardContext()); + parse(context, sortBuilder); sortBuilder = jsonBuilder(); sortBuilder.startObject(); @@ -136,9 +135,11 @@ public class SortParserTests extends ESSingleNodeTestCase { } protected void parse(TestSearchContext context, XContentBuilder sortBuilder) throws Exception { + QueryParseContext parseContext = context.getQueryShardContext().parseContext(); XContentParser parser = XContentHelper.createParser(sortBuilder.bytes()); + parser.setParseFieldMatcher(ParseFieldMatcher.STRICT); + parseContext.reset(parser); parser.nextToken(); - GeoDistanceSortParser geoParser = new GeoDistanceSortParser(); - geoParser.parse(parser, context.getQueryShardContext()); + GeoDistanceSortBuilder.PROTOTYPE.fromXContent(parseContext, null); } } diff --git a/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java index c423513ac72..15fa7e64e3f 100644 --- a/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java +++ b/core/src/test/java/org/elasticsearch/search/stats/SearchStatsUnitTests.java @@ -32,9 +32,9 @@ public class SearchStatsUnitTests extends ESTestCase { // let's create two dummy search stats with groups Map groupStats1 = new HashMap<>(); Map groupStats2 = new HashMap<>(); - groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1)); - SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1); - SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2); + groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)); + SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1); + SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2); // adding these two search stats and checking group stats are correct searchStats1.add(searchStats2); @@ -62,5 +62,8 @@ public class SearchStatsUnitTests extends ESTestCase { assertEquals(equalTo, stats.getScrollCount()); assertEquals(equalTo, stats.getScrollTimeInMillis()); assertEquals(equalTo, stats.getScrollCurrent()); + assertEquals(equalTo, stats.getSuggestCount()); + assertEquals(equalTo, stats.getSuggestTimeInMillis()); + assertEquals(equalTo, stats.getSuggestCurrent()); } } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java index ef0416df90d..152c450f6ec 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java @@ -32,7 +32,8 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Requests; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; @@ -41,6 +42,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.percolator.PercolatorFieldMapper; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.search.suggest.completion.CompletionSuggestion; @@ -198,9 +200,10 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg"). size(numDocs).payload(Collections.singletonList("count")); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", prefix).execute().actionGet(); - assertNoFailures(suggestResponse); - CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix)) + .execute().actionGet(); + assertNoFailures(searchResponse); + CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo"); CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); assertThat(options.getOptions().size(), equalTo(numDocs)); for (CompletionSuggestion.Entry.Option option : options) { @@ -219,9 +222,10 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { indexRandom(true, indexRequestBuilders); CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg") .payload(Collections.singletonList("test_field")); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", prefix).execute().actionGet(); - assertNoFailures(suggestResponse); - CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix)) + .execute().actionGet(); + assertNoFailures(searchResponse); + CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo"); CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); assertThat(options.getOptions().size(), equalTo(2)); for (CompletionSuggestion.Entry.Option option : options.getOptions()) { @@ -257,9 +261,10 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg") .payload(Arrays.asList("title", "count")); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", prefix).execute().actionGet(); - assertNoFailures(suggestResponse); - CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix)) + .execute().actionGet(); + assertNoFailures(searchResponse); + CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo"); List options = completionSuggestion.getEntries().get(0).getOptions(); assertThat(options.size(), equalTo(2)); assertThat(options.get(0).getText().toString(), equalTo("suggestion")); @@ -308,9 +313,10 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg") .size(suggestionSize).payload(payloadFields); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", prefix).execute().actionGet(); - assertNoFailures(suggestResponse); - CompletionSuggestion completionSuggestion = suggestResponse.getSuggest().getSuggestion("foo"); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix)) + .execute().actionGet(); + assertNoFailures(searchResponse); + CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo"); CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0); assertThat(options.getOptions().size(), equalTo(suggestionSize)); int id = numDocs; @@ -406,12 +412,12 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("testSuggestions", - new CompletionSuggestionBuilder(FIELD).text("test").size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("testSuggestions", new CompletionSuggestionBuilder(FIELD).text("test").size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, "testSuggestions", "testing"); - Suggest.Suggestion.Entry.Option option = suggestResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); + assertSuggestions(searchResponse, "testSuggestions", "testing"); + Suggest.Suggestion.Entry.Option option = searchResponse.getSuggest().getSuggestion("testSuggestions").getEntries().get(0).getOptions().get(0); assertThat(option, is(instanceOf(CompletionSuggestion.Entry.Option.class))); CompletionSuggestion.Entry.Option prefixOption = (CompletionSuggestion.Entry.Option) option; @@ -607,16 +613,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { .get(); assertThat(putMappingResponse.isAcknowledged(), is(true)); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("suggs", - SuggestBuilders.completionSuggestion(FIELD + ".suggest").text("f").size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("suggs", SuggestBuilders.completionSuggestion(FIELD + ".suggest").text("f").size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, "suggs"); + assertSuggestions(searchResponse, "suggs"); client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get(); ensureGreen(INDEX); - SuggestResponse afterReindexingResponse = client().prepareSuggest(INDEX).addSuggestion("suggs", - SuggestBuilders.completionSuggestion(FIELD + ".suggest").text("f").size(10) + SearchResponse afterReindexingResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("suggs", SuggestBuilders.completionSuggestion(FIELD + ".suggest").text("f").size(10)) ).execute().actionGet(); assertSuggestions(afterReindexingResponse, "suggs", "Foo Fighters"); } @@ -632,15 +638,15 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nirv").size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Nirv").size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nirw", Fuzziness.ONE).size(10) + searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Nirw", Fuzziness.ONE).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); } public void testThatFuzzySuggesterSupportsEditDistances() throws Exception { @@ -655,16 +661,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); // edit distance 1 - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Norw", Fuzziness.ONE).size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Norw", Fuzziness.ONE).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo"); + assertSuggestions(searchResponse, false, "foo"); // edit distance 2 - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Norw", Fuzziness.TWO).size(10) + searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Norw", Fuzziness.TWO).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); } public void testThatFuzzySuggesterSupportsTranspositions() throws Exception { @@ -678,15 +684,16 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nriv", FuzzyOptions.builder().setTranspositions(false).build()).size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", + SuggestBuilders.completionSuggestion(FIELD).prefix("Nriv", FuzzyOptions.builder().setTranspositions(false).build()).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo"); + assertSuggestions(searchResponse, false, "foo"); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nriv", Fuzziness.ONE).size(10) + searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("Nriv", Fuzziness.ONE).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); } public void testThatFuzzySuggesterSupportsMinPrefixLength() throws Exception { @@ -700,15 +707,17 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nriva", FuzzyOptions.builder().setFuzzyMinLength(6).build()).size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", + SuggestBuilders.completionSuggestion(FIELD).prefix("Nriva", FuzzyOptions.builder().setFuzzyMinLength(6).build()).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo"); + assertSuggestions(searchResponse, false, "foo"); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nrivan", FuzzyOptions.builder().setFuzzyMinLength(6).build()).size(10) + searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", + SuggestBuilders.completionSuggestion(FIELD).prefix("Nrivan", FuzzyOptions.builder().setFuzzyMinLength(6).build()).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); } public void testThatFuzzySuggesterSupportsNonPrefixLength() throws Exception { @@ -722,15 +731,17 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nirw", FuzzyOptions.builder().setFuzzyPrefixLength(4).build()).size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", + SuggestBuilders.completionSuggestion(FIELD).prefix("Nirw", FuzzyOptions.builder().setFuzzyPrefixLength(4).build()).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo"); + assertSuggestions(searchResponse, false, "foo"); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", - SuggestBuilders.completionSuggestion(FIELD).prefix("Nirvo", FuzzyOptions.builder().setFuzzyPrefixLength(4).build()).size(10) + searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion("foo", + SuggestBuilders.completionSuggestion(FIELD).prefix("Nirvo", FuzzyOptions.builder().setFuzzyPrefixLength(4).build()).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "Nirvana"); + assertSuggestions(searchResponse, false, "foo", "Nirvana"); } public void testThatFuzzySuggesterIsUnicodeAware() throws Exception { @@ -748,18 +759,18 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder completionSuggestionBuilder = SuggestBuilders.completionSuggestion(FIELD).prefix("öööи", FuzzyOptions.builder().setUnicodeAware(true).build()).size(10); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", completionSuggestionBuilder).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "ööööö"); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", completionSuggestionBuilder)).execute().actionGet(); + assertSuggestions(searchResponse, false, "foo", "ööööö"); // removing unicode awareness leads to no result completionSuggestionBuilder = SuggestBuilders.completionSuggestion(FIELD).prefix("öööи", FuzzyOptions.builder().setUnicodeAware(false).build()).size(10); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", completionSuggestionBuilder).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo"); + searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", completionSuggestionBuilder)).execute().actionGet(); + assertSuggestions(searchResponse, false, "foo"); // increasing edit distance instead of unicode awareness works again, as this is only a single character completionSuggestionBuilder = SuggestBuilders.completionSuggestion(FIELD).prefix("öööи", FuzzyOptions.builder().setUnicodeAware(false).setFuzziness(Fuzziness.TWO).build()).size(10); - suggestResponse = client().prepareSuggest(INDEX).addSuggestion("foo", completionSuggestionBuilder).execute().actionGet(); - assertSuggestions(suggestResponse, false, "foo", "ööööö"); + searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", completionSuggestionBuilder)).execute().actionGet(); + assertSuggestions(searchResponse, false, "foo", "ööööö"); } public void testThatStatsAreWorking() throws Exception { @@ -787,8 +798,8 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { refresh(); ensureGreen(); // load the fst index into ram - client().prepareSuggest(INDEX).addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("f")).get(); - client().prepareSuggest(INDEX).addSuggestion("foo", SuggestBuilders.completionSuggestion(otherField).prefix("f")).get(); + client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(FIELD).prefix("f"))).get(); + client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion(otherField).prefix("f"))).get(); // Get all stats IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(INDEX).setIndices(INDEX).setCompletion(true).get(); @@ -884,13 +895,11 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { } } - public void assertSuggestions(String suggestionName, SuggestionBuilder suggestBuilder, String... suggestions) { - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionName, suggestBuilder - ).execute().actionGet(); - assertSuggestions(suggestResponse, suggestionName, suggestions); - + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion(suggestionName, suggestBuilder)).execute().actionGet(); + assertSuggestions(searchResponse, suggestionName, suggestions); } + public void assertSuggestions(String suggestion, String... suggestions) { String suggestionName = RandomStrings.randomAsciiOfLength(random(), 10); CompletionSuggestionBuilder suggestionBuilder = SuggestBuilders.completionSuggestion(FIELD).text(suggestion).size(10); @@ -899,28 +908,29 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase { public void assertSuggestionsNotInOrder(String suggestString, String... suggestions) { String suggestionName = RandomStrings.randomAsciiOfLength(random(), 10); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionName, - SuggestBuilders.completionSuggestion(FIELD).text(suggestString).size(10) + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion(suggestionName, + SuggestBuilders.completionSuggestion(FIELD).text(suggestString).size(10)) ).execute().actionGet(); - assertSuggestions(suggestResponse, false, suggestionName, suggestions); + assertSuggestions(searchResponse, false, suggestionName, suggestions); } - static void assertSuggestions(SuggestResponse suggestResponse, String name, String... suggestions) { - assertSuggestions(suggestResponse, true, name, suggestions); + static void assertSuggestions(SearchResponse searchResponse, String name, String... suggestions) { + assertSuggestions(searchResponse, true, name, suggestions); } - private static void assertSuggestions(SuggestResponse suggestResponse, boolean suggestionOrderStrict, String name, String... suggestions) { - assertAllSuccessful(suggestResponse); + private static void assertSuggestions(SearchResponse searchResponse, boolean suggestionOrderStrict, String name, String... suggestions) { + assertAllSuccessful(searchResponse); List suggestionNames = new ArrayList<>(); - for (Suggest.Suggestion> suggestion : iterableAsArrayList(suggestResponse.getSuggest())) { + for (Suggest.Suggestion> suggestion : iterableAsArrayList(searchResponse.getSuggest())) { suggestionNames.add(suggestion.getName()); } String expectFieldInResponseMsg = String.format(Locale.ROOT, "Expected suggestion named %s in response, got %s", name, suggestionNames); - assertThat(expectFieldInResponseMsg, suggestResponse.getSuggest().getSuggestion(name), is(notNullValue())); + assertThat(expectFieldInResponseMsg, searchResponse.getSuggest().getSuggestion(name), is(notNullValue())); - Suggest.Suggestion> suggestion = suggestResponse.getSuggest().getSuggestion(name); + Suggest.Suggestion> suggestion = searchResponse.getSuggest().getSuggestion(name); List suggestionList = getNames(suggestion.getEntries().get(0)); List options = suggestion.getEntries().get(0).getOptions(); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java index 58458c9d244..ee45a9d0371 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/ContextCompletionSuggestSearchIT.java @@ -23,7 +23,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomStrings; import org.apache.lucene.spatial.util.GeoHashUtils; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.suggest.SuggestResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; @@ -624,16 +624,17 @@ public class ContextCompletionSuggestSearchIT extends ESIntegTestCase { String suggestionName = randomAsciiOfLength(10); CompletionSuggestionBuilder context = SuggestBuilders.completionSuggestion(FIELD).text("h").size(10) .contexts(Collections.singletonMap("st", Collections.singletonList(GeoQueryContext.builder().setGeoPoint(new GeoPoint(52.52, 13.4)).build()))); - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionName, context).get(); + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion(suggestionName, context)).get(); - assertEquals(suggestResponse.getSuggest().size(), 1); - assertEquals("Hotel Amsterdam in Berlin", suggestResponse.getSuggest().getSuggestion(suggestionName).iterator().next().getOptions().iterator().next().getText().string()); + assertEquals(searchResponse.getSuggest().size(), 1); + assertEquals("Hotel Amsterdam in Berlin", searchResponse.getSuggest().getSuggestion(suggestionName).iterator().next().getOptions().iterator().next().getText().string()); } public void assertSuggestions(String suggestionName, SuggestionBuilder suggestBuilder, String... suggestions) { - SuggestResponse suggestResponse = client().prepareSuggest(INDEX).addSuggestion(suggestionName, suggestBuilder + SearchResponse searchResponse = client().prepareSearch(INDEX).suggest( + new SuggestBuilder().addSuggestion(suggestionName, suggestBuilder) ).execute().actionGet(); - CompletionSuggestSearchIT.assertSuggestions(suggestResponse, suggestionName, suggestions); + CompletionSuggestSearchIT.assertSuggestions(searchResponse, suggestionName, suggestions); } private void createIndexAndMapping(CompletionMappingBuilder completionMappingBuilder) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java index 25b0e06c4f1..fc0a8abac27 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java +++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolStatsTests.java @@ -65,7 +65,6 @@ public class ThreadPoolStatsTests extends ESTestCase { try (BytesStreamOutput os = new BytesStreamOutput()) { List stats = new ArrayList<>(); - stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SUGGEST, -1, 0, 0, 0, 0, 0L)); stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, -1, 0, 0, 0, 0, 0L)); stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.WARMER, -1, 0, 0, 0, 0, 0L)); stats.add(new ThreadPoolStats.Stats(ThreadPool.Names.GENERIC, -1, 0, 0, 0, 0, 0L)); @@ -104,7 +103,6 @@ public class ThreadPoolStatsTests extends ESTestCase { ThreadPool.Names.GENERIC, ThreadPool.Names.SAME, ThreadPool.Names.SEARCH, - ThreadPool.Names.SUGGEST, ThreadPool.Names.WARMER)); } } diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle index 2ab78fe7e41..177ab518a10 100644 --- a/distribution/rpm/build.gradle +++ b/distribution/rpm/build.gradle @@ -25,7 +25,14 @@ task buildRpm(type: Rpm) { packageGroup 'Application/Internet' prefix '/usr' packager 'Elasticsearch' - release '1' + if (version.contains('~')) { + def tokenized = version.tokenize('~') + version tokenized[0] + release tokenized[1] + } else { + version version + release '1' + } arch NOARCH os LINUX // TODO ospackage doesn't support icon but we used to have one diff --git a/distribution/src/main/resources/bin/elasticsearch.bat b/distribution/src/main/resources/bin/elasticsearch.bat index 4da9a5d032a..a2e5d45e1c4 100644 Binary files a/distribution/src/main/resources/bin/elasticsearch.bat and b/distribution/src/main/resources/bin/elasticsearch.bat differ diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 007018a6abc..8780b3a710e 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -189,7 +189,7 @@ The following settings are supported: Big files can be broken down into chunks during snapshotting if needed. The chunk size can be specified in bytes or by using size value notation, - i.e. `1g`, `10m`, `5k`. Defaults to `100m`. + i.e. `1gb`, `10mb`, `5kb`. Defaults to `1gb`. `compress`:: @@ -210,7 +210,7 @@ The following settings are supported: to split the chunk into several parts, each of `buffer_size` length, and to upload each part in its own request. Note that setting a buffer size lower than `5mb` is not allowed since it will prevents the use of the - Multipart API and may result in upload errors. Defaults to `5mb`. + Multipart API and may result in upload errors. Defaults to `100mb`. `max_retries`:: diff --git a/docs/reference/indices/stats.asciidoc b/docs/reference/indices/stats.asciidoc index 87eda44e104..e990a7ff6bd 100644 --- a/docs/reference/indices/stats.asciidoc +++ b/docs/reference/indices/stats.asciidoc @@ -34,7 +34,8 @@ specified as well in the URI. Those stats can be any of: `get`:: Get statistics, including missing stats. -`search`:: Search statistics. You can include statistics for custom groups by adding +`search`:: Search statistics including suggest statistics. + You can include statistics for custom groups by adding an extra `groups` parameter (search operations can be associated with one or more groups). The `groups` parameter accepts a comma separated list of group names. Use `_all` to return statistics for all groups. @@ -47,7 +48,6 @@ specified as well in the URI. Those stats can be any of: `merge`:: Merge statistics. `request_cache`:: <> statistics. `refresh`:: Refresh statistics. -`suggest`:: Suggest statistics. `warmer`:: Warmer statistics. `translog`:: Translog statistics. diff --git a/docs/reference/migration/migrate_5_0/index-apis.asciidoc b/docs/reference/migration/migrate_5_0/index-apis.asciidoc index 72651295bbc..b820194c2d0 100644 --- a/docs/reference/migration/migrate_5_0/index-apis.asciidoc +++ b/docs/reference/migration/migrate_5_0/index-apis.asciidoc @@ -45,4 +45,7 @@ longer a double and is instead an object encapsulating the one-minute, five-minute and fifteen-minute load averages. Additionally, the recent CPU usage can be obtained from `OsStats.Cpu#getPercent`. +==== Suggest stats +Suggest stats exposed through `suggest` in indices stats has been merged +with `search` stats. `suggest` stats is exposed as part of `search` stats. diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc index 59f0b1c15e2..df526740f33 100644 --- a/docs/reference/migration/migrate_5_0/java.asciidoc +++ b/docs/reference/migration/migrate_5_0/java.asciidoc @@ -22,6 +22,25 @@ can be replaced with client.prepareSearch(indices).setSource(new SearchSourceBuilder().size(0).query(query)).get(); ----- +==== Suggest api has been removed + +The suggest api has been removed from the Java api, use the suggest option in search api, it has been optimized +for suggest-only request. + +The following call + +[source,java] +----- +client.prepareSuggest(indices).addSuggestion("foo", SuggestBuilders.completionSuggestion("field").text("s")).get(); +----- + +can be replaced with + +[source,java] +----- +client.prepareSearch(indices).suggest(new SuggestBuilder().addSuggestion("foo", SuggestBuilders.completionSuggestion("field").text("s"))).get(); +----- + ==== Elasticsearch will no longer detect logging implementations Elasticsearch now logs only to log4j 1.2. Previously if log4j wasn't on the diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc index 10268887417..46014e3bcd4 100644 --- a/docs/reference/migration/migrate_5_0/plugins.asciidoc +++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc @@ -6,6 +6,10 @@ structure of the plugin ZIP archive has changed. All the plugin files must be contained in a top-level directory called `elasticsearch`. If you use the gradle build, this structure is automatically generated. +==== Plugins isolation + +`isolated` option has been removed. Each plugin will have its own classloader. + ==== Site plugins removed Site plugins have been removed. Site plugins should be reimplemented as Kibana diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc index bf242a9929f..f9b104e2fcf 100644 --- a/docs/reference/migration/migrate_5_0/settings.asciidoc +++ b/docs/reference/migration/migrate_5_0/settings.asciidoc @@ -71,6 +71,10 @@ type for any thread pool has been removed. It is still possible to adjust relevant thread pool parameters for each of the thread pools (e.g., depending on the thread pool type, `keep_alive`, `queue_size`, etc.). +==== Threadpool settings + +The `suggest` threadpool has been removed, now suggest requests use the +`search` threadpool. ==== Analysis settings diff --git a/docs/reference/modules.asciidoc b/docs/reference/modules.asciidoc index 5ef8a41d3f5..b71d1224e7e 100644 --- a/docs/reference/modules.asciidoc +++ b/docs/reference/modules.asciidoc @@ -45,6 +45,10 @@ The modules in this section are: <>:: A Java node client joins the cluster, but doesn't hold data or act as a master node. + +<>:: + + A built-in scripting language for Elasticsearch that's designed to be as secure as possible. <>:: @@ -53,7 +57,8 @@ The modules in this section are: <>:: Custom scripting available in Lucene Expressions, Groovy, Python, and - Javascript. + Javascript. You can also write scripts in the built-in scripting language, + <>. <>:: @@ -89,6 +94,8 @@ include::modules/network.asciidoc[] include::modules/node.asciidoc[] +include::modules/painless.asciidoc[] + include::modules/plugins.asciidoc[] include::modules/scripting.asciidoc[] diff --git a/docs/reference/modules/painless.asciidoc b/docs/reference/modules/painless.asciidoc new file mode 100644 index 00000000000..7c2bcdd9750 --- /dev/null +++ b/docs/reference/modules/painless.asciidoc @@ -0,0 +1,693 @@ +[[modules-scripting-painless]] +== Painless Scripting Language + +_Painless_ is a simple, secure scripting language built in to Elasticsearch as a module. +It is designed specifically for use with Elasticsearch and can safely be used dynamically. + +A Painless script is essentially a single function. Painless does not provide support +for defining multiple functions within a script. The Painless syntax is similar to +http://groovy-lang.org/index.html[Groovy]. + +You can use Painless anywhere a script can be used in Elasticsearch--simply set the `lang` parameter +to `painless`. + +[[painless-features]] +[float] +=== Painless Features + +* Control flow: `for` loops, `while` loops, `do/while` loops, `if/else` + +* Fully Typed: all available types/methods described in <> + +* Arithmetic operators: multiplication `*`, division `/`, addition `+`, subtraction `-`, precedence `( )` + +* Comparison operators: less than `<`, less than or equal to `<=`, greater than `>`, greater than or equal to `>=`, equal to `==`, and not equal to `!=`, reference equals `===`, reference not equals `!==` + +* Boolean operators: not `!`, and `&&`, or `||` + +* Bitwise operators: shift left `<<`, shift right `>>`, unsigned shift `>>>`, and `&`, or `|`, xor `^`, not `~` + +* Shortcuts for list, map access using the dot `.` operator + + +[[painless-examples]] +[float] +=== Painless Examples + +To illustrate how Painless works, let's load some hockey stats into an Elasticsearch index: + +[source,sh] +---------------------------------------------------------------- +curl -XDELETE http://localhost:9200/hockey-stats +curl -XPUT http://localhost:9200/hockey-stats +curl -XPUT http://localhost:9200/hockey-stats/player/1 -d '{"first":"johnny", "last":"gaudreau", "goals":[9, 27, 1], "assists":[17, 46, 0], "gp":[26, 82, 1]}' +curl -XPUT http://localhost:9200/hockey-stats/player/2 -d '{"first":"sean", "last":"monohan", "goals":[7, 54, 26], "assists":[11, 26, 13], "gp":[26, 82, 82]}' +curl -XPUT http://localhost:9200/hockey-stats/player/3 -d '{"first":"jiri", "last":"hudler", "goals":[5, 34, 36], "assists":[11, 62, 42], "gp":[24, 80, 79]}' +curl -XPUT http://localhost:9200/hockey-stats/player/4 -d '{"first":"micheal", "last":"frolik", "goals":[4, 6, 15], "assists":[8, 23, 15], "gp":[26, 82, 82]}' +curl -XPUT http://localhost:9200/hockey-stats/player/5 -d '{"first":"sam", "last":"bennett", "goals":[5, 0, 0], "assists":[8, 1, 0], "gp":[26, 1, 0]}' +curl -XPUT http://localhost:9200/hockey-stats/player/6 -d '{"first":"dennis", "last":"wideman", "goals":[0, 26, 15], "assists":[11, 30, 24], "gp":[26, 81, 82]}' +curl -XPUT http://localhost:9200/hockey-stats/player/7 -d '{"first":"david", "last":"jones", "goals":[7, 19, 5], "assists":[3, 17, 4], "gp":[26, 45, 34]}' +curl -XPUT http://localhost:9200/hockey-stats/player/8 -d '{"first":"tj", "last":"brodie", "goals":[2, 14, 7], "assists":[8, 42, 30], "gp":[26, 82, 82]}' +curl -XPUT http://localhost:9200/hockey-stats/player/9 -d '{"first":"mark", "last":"giordano", "goals":[6, 30, 15], "assists":[3, 30, 24], "gp":[26, 60, 63]}' +curl -XPUT http://localhost:9200/hockey-stats/player/10 -d '{"first":"mikael", "last":"backlund", "goals":[3, 15, 13], "assists":[6, 24, 18], "gp":[26, 82, 82]}' +curl -XPUT http://localhost:9200/hockey-stats/player/11 -d '{"first":"joe", "last":"colborne", "goals":[3, 18, 13], "assists":[6, 20, 24], "gp":[26, 67, 82]}' +---------------------------------------------------------------- + +[float] +==== Accessing Doc Values from Painless + +All Painless scripts take in a `Map` of values called `input`. Document values can be accessed through another `Map` within the `input` variable. + +For example, the following script calculates a player's total goals. This example uses a strongly typed `int` and a `for` loop. + +[source,sh] +---------------------------------------------------------------- +curl -XGET http://localhost:9200/hockey-stats/_search -d '{ + "query": { + "function_score": { + "script_score" : { + "script" : { + "inline": + "int total = 0; for (int i = 0; i < input.doc.goals.size(); ++i) { total += input.doc.goals[i]; } return total;", + "lang": "painless" + } + } + } + } +}' +---------------------------------------------------------------- + +Alternatively, you could do the same thing using a script field instead of a function score: + +[source,sh] +---------------------------------------------------------------- +curl -XGET http://localhost:9200/hockey-stats/_search -d '{ + "query": { + "match_all": {}}, + "script_fields": { + "total_goals": { + "script": { + "inline": "int total = 0; for (int i = 0; i < input.doc.goals.size(); ++i) { total += input.doc.goals[i]; } return total;", + "lang": "painless" + } + } + } +}' +---------------------------------------------------------------- + +You must always specify the index of the field value you want, even if there's only a single item in the field. +All fields in Elasticsearch are multi-valued and Painless does not provide a `.value` shortcut. The following example uses a Painless script to sort the players by their combined first and last names. The names are accessed using +`input.doc.first.0` and `input.doc.last.0`. + +[source,sh] +---------------------------------------------------------------- +curl -XGET http://localhost:9200/hockey-stats/_search -d '{ + "query" : { + "match_all": {}}, + "sort" : { + "_script" : { + "type" : "string", + "script" : {"inline": "input.doc.first.0 + \" \" + input.doc.last.0", + "lang": "painless"}, + "order" : "asc" + } + } +}' +---------------------------------------------------------------- + +[float] +==== Updating Fields with Painless + +You can also easily update fields. You access the original source for a field as `input.ctx._source.`. + +First, let's look at the source data for a player by submitting the following request: + +[source,sh] +---------------------------------------------------------------- +curl -XGET http://localhost:9200/hockey-stats/_search -d '{ + "fields" : ["_id", "_source"], "query" : { + "term" : { "_id" : 1 } + } +}' +---------------------------------------------------------------- + +To change player 1's last name to _hockey_, simply set `input.ctx._source.last` to the new value: + +[source,sh] +---------------------------------------------------------------- +curl -XPOST http://localhost:9200/hockey-stats/player/1/_update -d '{ + "script": { + "inline": "input.ctx._source.last = input.last", + "params": {"last": "hockey"}, + "lang": "painless" + } +}' +---------------------------------------------------------------- + +You can also add fields to a document. For example, this script adds a new field that contains +the player's nickname, _hockey_. + +[source,sh] +---------------------------------------------------------------- +curl -XPOST http://localhost:9200/hockey-stats/player/1/_update -d '{ + "script": { + "inline": "input.ctx._source.last = input.last input.ctx._source.nick = input.nick", + "params": {"last": "gaudreau", "nick": "hockey"}, + "lang": "painless" + } +}' +---------------------------------------------------------------- + +[float] +==== Writing Type-Safe Scripts to Improve Performance + +If you explicitly specify types, the compiler doesn't have to perform type lookups at runtime, which can significantly +improve performance. For example, the following script performs the same first name, last name sort we showed before, +but it's fully type-safe. + +[source,sh] +---------------------------------------------------------------- +curl -XGET http://localhost:9200/hockey-stats/_search -d '{ + "query": { + "match_all": {} + }, + "script_fields": { + "full_name_dynamic": { + "script": { + "inline": "def first = input.doc.first.0; def last = input.doc.last.0; return first + \" \" + last;", + "lang": "painless" + } + }, + "full_name_static": { + "script": { + "inline": + "String first = (String)((List)((Map)input.get(\"doc\")).get(\"first\")).get(0); String last = (String)((List)((Map)input.get(\"doc\")).get(\"last\")).get(0); return first + \" \" + last;", + "lang": "painless" + } + } + } +}' +---------------------------------------------------------------- + +[[painless-api]] +[float] +=== Painless API + +The following types are available for use in the Painless language. Most types and methods map directly to their Java equivalents--for more information, see the corresponding https://docs.oracle.com/javase/8/docs/api/java/lang/package-summary.html[Javadoc]. + + +[float] +==== Dynamic Types + +`def` (This type can be used to represent any other type.) + +[float] +==== Basic Types + +`void` + +`boolean` + +`short` + +`char` + +`int` + +`long` + +`float` + +`double` + +[float] +==== Complex Types + +Non-static methods/members in superclasses are available to subclasses. +Generic types with unspecified generic parameters are parameters of type `def`. + +----- +ArithmeticException extends Exception + () +----- + +----- +ArrayList extends List + () +----- + +----- +ArrayList extends List + () +----- + +----- +ArrayList extends List + () +----- + +----- +Boolean extends Object + (boolean) + static Boolean valueOf(boolean) + boolean booleanValue() +----- + +----- +Character extends Object + (char) + static Character valueOf(char) + char charValue() + static char MIN_VALUE + static char MAX_VALUE +----- + +----- +CharSequence extends Object + char charAt(int) + int length() +----- + +----- +Collection extends Object + boolean add(def) + void clear() + boolean contains(def) + boolean isEmpty() + Iterator iterator() + boolean remove(def) + int size() +----- + +----- +Collection extends Object + boolean add(Object) + void clear() + boolean contains(Object) + boolean isEmpty() + Iterator iterator() + boolean remove(Object) + int size() +----- + +----- +Collection extends Object + boolean add(String) + void clear() + boolean contains(String) + boolean isEmpty() + Iterator iterator() + boolean remove(String) + int size() +----- + +----- +Double extends Number + (double) + static Double valueOf(double) + static double MIN_VALUE + static double MAX_VALUE +----- + +----- +Exception extends Object + String getMessage() +----- + +----- +Float extends Number + (float) + static Float valueOf(float) + static float MIN_VALUE + static float MAX_VALUE +----- + +----- +HashMap extends Map + () +----- + +----- +HashMap extends Map + () +----- + +----- +HashMap extends Map + () +----- + +----- +HashMap extends Map + () +----- + +----- +IllegalArgument extends Exception + () +----- + +----- +IllegalState extends Exception + () +----- + +----- +Integer extends Number + (int) + static Integer valueOf(int) + static int MIN_VALUE + static int MAX_VALUE +----- + +----- +Iterator extends Object + boolean hasNext() + def next() + void remove() +----- + +----- +Iterator extends Object + boolean hasNext() + String next() + void remove() +----- + +----- +List extends Collection + def set(int, def) + def get(int) + def remove(int) +----- + +----- +List extends Collection + Object set(int, Object) + Object get(int) + Object remove(int) +----- + +----- +List extends Collection + String set(int, String) + String get(int) + String remove(int) +----- + +----- +Long extends Number + (long) + static Long valueOf(long) + static long MIN_VALUE + static long MAX_VALUE +----- + +----- +Map extends Object + def put (def, def) + def get (def) + def remove (def) + boolean isEmpty() + int size() + boolean containsKey(def) + boolean containsValue(def) + Set keySet() + Collection values() +----- + +----- +Map extends Object + Object put (Object, Object) + Object get (Object) + Object remove (Object) + boolean isEmpty() + int size() + boolean containsKey(Object) + boolean containsValue(Object) + Set keySet() + Collection values() +----- + +----- +Map extends Object + def put (String, def) + def get (String) + def remove (String) + boolean isEmpty() + int size() + boolean containsKey(String) + boolean containsValue(def) + Set keySet() + Collection values() +----- + +----- +Map extends Object + Object put (String, Object) + Object get (String) + Object remove (String) + boolean isEmpty() + int size() + boolean containsKey(String) + boolean containsValue(Object) + Set keySet() + Collection values() +----- + +----- +Number extends Object + short shortValue() + short shortValue() + int intValue() + long longValue() + float floatValue() + double doubleValue() +----- + +----- +Object + String toString() + boolean equals(Object) + int hashCode() +----- + +----- +Set extends Collection +----- + +----- +Set extends Collection +----- + +----- +Set extends Collection +----- + +----- +Short extends Number + (short) + static Short valueOf(short) + static short MIN_VALUE + static short MAX_VALUE +----- + +----- +String extends CharSequence + (String) + int codePointAt(int) + int compareTo(String) + String concat(String) + boolean endsWith(String) + int indexOf(String, int) + boolean isEmpty() + String replace(CharSequence, CharSequence) + boolean startsWith(String) + String substring(int, int) + char[] toCharArray() + String trim() +----- + +----- +NumberFormatException extends Exception + () +----- + +----- +Void extends Object +----- + +[float] +==== Utility Classes + +----- +Math + static double abs(double) + static float fabs(float) + static long labs(long) + static int iabs(int) + static double acos(double) + static double asin(double) + static double atan(double) + static double atan2(double) + static double cbrt(double) + static double ceil(double) + static double cos(double) + static double cosh(double) + static double exp(double) + static double expm1(double) + static double floor(double) + static double hypt(double, double) + static double abs(double) + static double log(double) + static double log10(double) + static double log1p(double) + static double max(double, double) + static float fmax(float, float) + static long lmax(long, long) + static int imax(int, int) + static double min(double, double) + static float fmin(float, float) + static long lmin(long, long) + static int imin(int, int) + static double pow(double, double) + static double random() + static double rint(double) + static long round(double) + static double sin(double) + static double sinh(double) + static double sqrt(double) + static double tan(double) + static double tanh(double) + static double toDegrees(double) + static double toRadians(double) +----- + +----- +Utility + static boolean NumberToboolean(Number) + static char NumberTochar(Number) + static Boolean NumberToBoolean(Number) + static Short NumberToShort(Number) + static Character NumberToCharacter(Number) + static Integer NumberToInteger(Number) + static Long NumberToLong(Number) + static Float NumberToFloat(Number) + static Double NumberToDouble(Number) + static byte booleanTobyte(boolean) + static short booleanToshort(boolean) + static char booleanTochar(boolean) + static int booleanToint(boolean) + static long booleanTolong(boolean) + static float booleanTofloat(boolean) + static double booleanTodouble(boolean) + static Integer booleanToInteger(boolean) + static byte BooleanTobyte(Boolean) + static short BooleanToshort(Boolean) + static char BooleanTochar(Boolean) + static int BooleanToint(Boolean) + static long BooleanTolong(Boolean) + static float BooleanTofloat(Boolean) + static double BooleanTodouble(Boolean) + static Byte BooleanToByte(Boolean) + static Short BooleanToShort(Boolean) + static Character BooleanToCharacter(Boolean) + static Integer BooleanToInteger(Boolean) + static Long BooleanToLong(Boolean) + static Float BooleanToFloat(Boolean) + static Double BooleanToDouble(Boolean) + static boolean byteToboolean(byte) + static Short byteToShort(byte) + static Character byteToCharacter(byte) + static Integer byteToInteger(byte) + static Long byteToLong(byte) + static Float byteToFloat(byte) + static Double byteToDouble(byte) + static boolean ByteToboolean(Byte) + static char ByteTochar(Byte) + static boolean shortToboolean(short) + static Byte shortToByte(short) + static Character shortToCharacter(short) + static Integer shortToInteger(short) + static Long shortToLong(short) + static Float shortToFloat(short) + static Double shortToDouble(short) + static boolean ShortToboolean(Short) + static char ShortTochar(Short) + static boolean charToboolean(char) + static Byte charToByte(char) + static Short charToShort(char) + static Integer charToInteger(char) + static Long charToLong(char) + static Float charToFloat(char) + static Double charToDouble(char) + static boolean CharacterToboolean(Character) + static byte CharacterTobyte(Character) + static short CharacterToshort(Character) + static int CharacterToint(Character) + static long CharacterTolong(Character) + static float CharacterTofloat(Character) + static double CharacterTodouble(Character) + static Boolean CharacterToBoolean(Character) + static Byte CharacterToByte(Character) + static Short CharacterToShort(Character) + static Integer CharacterToInteger(Character) + static Long CharacterToLong(Character) + static Float CharacterToFloat(Character) + static Double CharacterToDouble(Character) + static boolean intToboolean(int) + static Byte intToByte(int) + static Short intToShort(int) + static Character intToCharacter(int) + static Long intToLong(int) + static Float intToFloat(int) + static Double intToDouble(int) + static boolean IntegerToboolean(Integer) + static char IntegerTochar(Integer) + static boolean longToboolean(long) + static Byte longToByte(long) + static Short longToShort(long) + static Character longToCharacter(long) + static Integer longToInteger(long) + static Float longToFloat(long) + static Double longToDouble(long) + static boolean LongToboolean(Long) + static char LongTochar(Long) + static boolean floatToboolean(float) + static Byte floatToByte(float) + static Short floatToShort(float) + static Character floatToCharacter(float) + static Integer floatToInteger(float) + static Long floatToLong(float) + static Double floatToDouble(float) + static boolean FloatToboolean(Float) + static char FloatTochar(Float) + static boolean doubleToboolean(double) + static Byte doubleToByte(double) + static Short doubleToShort(double) + static Character doubleToCharacter(double) + static Integer doubleToInteger(double) + static Long doubleToLong(double) + static Float doubleToFloat(double) + static boolean DoubleToboolean(Double) + static char DoubleTochar(Double) +----- + +----- +Def + static boolean defToboolean(def) + static byte defTobyte(def) + static short defToshort(def) + static char defTochar(def) + static int defToint(def) + static long defTolong(def) + static float defTofloat(def) + static double defTodouble(def) + static Boolean defToBoolean(def) + static Byte defToByte(def) + static Character defToCharacter(def) + static Integer defToInteger(def) + static Long defToLong(def) + static Float defToFloat(def) + static Double defToDouble(def) +----- \ No newline at end of file diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc index f4374a0f9b3..114740306ee 100644 --- a/docs/reference/modules/scripting.asciidoc +++ b/docs/reference/modules/scripting.asciidoc @@ -3,4 +3,3 @@ include::scripting/scripting.asciidoc[] include::scripting/advanced-scripting.asciidoc[] include::scripting/security.asciidoc[] - diff --git a/docs/reference/modules/scripting/scripting.asciidoc b/docs/reference/modules/scripting/scripting.asciidoc index 4f9d84f34f8..12524407900 100644 --- a/docs/reference/modules/scripting/scripting.asciidoc +++ b/docs/reference/modules/scripting/scripting.asciidoc @@ -1,27 +1,48 @@ [[modules-scripting]] == Scripting -The scripting module allows to use scripts in order to evaluate custom -expressions. For example, scripts can be used to return "script fields" -as part of a search request, or can be used to evaluate a custom score -for a query and so on. +The scripting module enables you to use scripts to evaluate custom +expressions. For example, you could use a script to return "script fields" +as part of a search request or evaluate a custom score for a query. -The scripting module uses by default http://groovy-lang.org/[groovy] -(previously http://mvel.codehaus.org/[mvel] in 1.3.x and earlier) as the -scripting language with some extensions. Groovy is used since it is extremely -fast and very simple to use. +TIP: Elasticsearch now has a built-in scripting language called _Painless_ +that provides a more secure alternative for implementing +scripts for Elasticsearch. We encourage you to try it out-- +for more information, see <>. + +The default scripting language is http://groovy-lang.org/[groovy] +(http://mvel.codehaus.org/[mvel] was the default in 1.3.x and earlier). + +Additional `lang` plugins enable you to run scripts written in other languages. +Everywhere a script can be used, you can include a `lang` parameter +to specify the language of the script. Plugins are available for following languages: + +[cols="<,<,<",options="header",] +|======================================================================= +|Language |Sandboxed |Required plugin +|groovy |no |built-in +|expression |yes |built-in +|mustache |yes |built-in +/painless /yes /built-in (module) +|javascript |no |{plugins}/lang-javascript.html[elasticsearch-lang-javascript] +|python |no |{plugins}/lang-python.html[elasticsearch-lang-python] +|======================================================================= .Groovy dynamic scripting off by default from v1.4.3 [IMPORTANT] =================================================== -Groovy dynamic scripting is off by default, preventing dynamic Groovy scripts -from being accepted as part of a request or retrieved from the special -`.scripts` index. You will still be able to use Groovy scripts stored in files -in the `config/scripts/` directory on every node. +Groovy dynamic scripting is off by default. This prevents Groovy scripts +from being accepted as part of a request or retrieved from the +`.scripts` index. You can still use Groovy file scripts stored in +the `config/scripts/` directory on every node. -To convert an inline script to a file, take this simple script -as an example: +To convert an inline script to a file-based script, save the contents +of the `inline` field to a file with the `.groovy` extension and +store it in the `config/scripts` directory on every data node in your +cluster. + +For example, if you have the following inline script: [source,js] ----------------------------------- @@ -38,15 +59,9 @@ GET /_search } ----------------------------------- -Save the contents of the `inline` field as a file called `config/scripts/my_script.groovy` -on every data node in the cluster: +Save `1 + my_var` in a file called `config/scripts/my_script.groovy`. -[source,js] ------------------------------------ -1 + my_var ------------------------------------ - -Now you can access the script by file name (without the extension): +To use the script in a request, specify its name (without the `.groovy` extension) in the `file` field: [source,js] ----------------------------------- @@ -67,21 +82,8 @@ GET /_search =================================================== - -Additional `lang` plugins are provided to allow to execute scripts in -different languages. All places where a script can be used, a `lang` parameter -can be provided to define the language of the script. The following are the -supported scripting languages: - -[cols="<,<,<",options="header",] -|======================================================================= -|Language |Sandboxed |Required plugin -|groovy |no |built-in -|expression |yes |built-in -|mustache |yes |built-in -|javascript |no |{plugins}/lang-javascript.html[elasticsearch-lang-javascript] -|python |no |{plugins}/lang-python.html[elasticsearch-lang-python] -|======================================================================= +[float] +=== File-based Scripts To increase security, Elasticsearch does not allow you to specify scripts for non-sandboxed languages with a request. Instead, scripts must be placed in the @@ -219,8 +221,6 @@ Indexed scripts can be deleted by: curl -XDELETE localhost:9200/_scripts/groovy/indexedCalculateScore ----------------------------------- - - [float] [[enable-dynamic-scripting]] === Enabling dynamic scripting diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc index bfd5474183c..199b6a9b88c 100644 --- a/docs/reference/modules/threadpool.asciidoc +++ b/docs/reference/modules/threadpool.asciidoc @@ -19,15 +19,10 @@ There are several thread pools, but the important ones include: queue_size of `200`. `search`:: - For count/search operations. Thread pool type is `fixed` + For count/search/suggest operations. Thread pool type is `fixed` with a size of `int((# of available_processors * 3) / 2) + 1`, queue_size of `1000`. -`suggest`:: - For suggest operations. Thread pool type is `fixed` - with a size of `# of available processors`, - queue_size of `1000`. - `get`:: For get operations. Thread pool type is `fixed` with a size of `# of available processors`, diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc index f8e08aaada9..dc986bf14d9 100644 --- a/docs/reference/query-dsl/bool-query.asciidoc +++ b/docs/reference/query-dsl/bool-query.asciidoc @@ -141,3 +141,10 @@ GET _search } --------------------------------- // AUTOSENSE + +==== Using named queries to see which clauses matched + +If you need to know which of the clauses in the bool query matched the documents +returned from the query, you can use +<> to assign a name to +each clause. diff --git a/modules/build.gradle b/modules/build.gradle index 3cafe7d903f..558163a420a 100644 --- a/modules/build.gradle +++ b/modules/build.gradle @@ -35,12 +35,6 @@ subprojects { throw new InvalidModelException("Modules cannot contain config files") } - project.afterEvaluate { - if (esplugin.isolated == false) { - throw new InvalidModelException("Modules cannot disable isolation") - } - } - // these are implementation details of our build, no need to publish them! install.enabled = false uploadArchives.enabled = false diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java index 29bd7f47498..c8b30a5e2de 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java @@ -70,8 +70,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.action.suggest.SuggestAction; -import org.elasticsearch.action.suggest.SuggestRequest; import org.elasticsearch.action.termvectors.MultiTermVectorsAction; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsAction; @@ -88,7 +86,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.groovy.GroovyPlugin; import org.elasticsearch.search.action.SearchTransportService; -import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -440,17 +437,6 @@ public class IndicesRequestTests extends ESIntegTestCase { assertSameIndices(indicesStatsRequest, indicesStats); } - public void testSuggest() { - String suggestAction = SuggestAction.NAME + "[s]"; - interceptTransportActions(suggestAction); - - SuggestRequest suggestRequest = new SuggestRequest(randomIndicesOrAliases()).suggest(new SuggestBuilder()); - internalCluster().clientNodeClient().suggest(suggestRequest).actionGet(); - - clearInterceptedActions(); - assertSameIndices(suggestRequest, suggestAction); - } - public void testValidateQuery() { String validateQueryShardAction = ValidateQueryAction.NAME + "[s]"; interceptTransportActions(validateQueryShardAction); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java index 94f60f8802c..79cac3181ea 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java @@ -60,8 +60,6 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.action.suggest.SuggestRequestBuilder; -import org.elasticsearch.action.suggest.SuggestResponse; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -1269,34 +1267,17 @@ public class SuggestSearchTests extends ESIntegTestCase { } protected Suggest searchSuggest(String suggestText, int expectShardsFailed, Map> suggestions) { - if (randomBoolean()) { - SearchRequestBuilder builder = client().prepareSearch().setSize(0); - SuggestBuilder suggestBuilder = new SuggestBuilder(); - if (suggestText != null) { - suggestBuilder.setGlobalText(suggestText); - } - for (Entry> suggestion : suggestions.entrySet()) { - suggestBuilder.addSuggestion(suggestion.getKey(), suggestion.getValue()); - } - builder.suggest(suggestBuilder); - SearchResponse actionGet = builder.execute().actionGet(); - assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed)); - return actionGet.getSuggest(); - } else { - SuggestRequestBuilder builder = client().prepareSuggest(); - if (suggestText != null) { - builder.setSuggestText(suggestText); - } - for (Entry> suggestion : suggestions.entrySet()) { - builder.addSuggestion(suggestion.getKey(), suggestion.getValue()); - } - - SuggestResponse actionGet = builder.execute().actionGet(); - assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed)); - if (expectShardsFailed > 0) { - throw new SearchPhaseExecutionException("suggest", "Suggest execution failed", new ShardSearchFailure[0]); - } - return actionGet.getSuggest(); + SearchRequestBuilder builder = client().prepareSearch().setSize(0); + SuggestBuilder suggestBuilder = new SuggestBuilder(); + if (suggestText != null) { + suggestBuilder.setGlobalText(suggestText); } + for (Entry> suggestion : suggestions.entrySet()) { + suggestBuilder.addSuggestion(suggestion.getKey(), suggestion.getValue()); + } + builder.suggest(suggestBuilder); + SearchResponse actionGet = builder.execute().actionGet(); + assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed)); + return actionGet.getSuggest(); } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index a1564aa136a..84b7ab58ef2 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -354,7 +354,7 @@ public abstract class AbstractAsyncBulkByScrollAction indexingFailures, List searchFailures, boolean timedOut) { - if (task.isCancelled() || false == mainRequest.isRefresh()) { + if (task.isCancelled() || false == mainRequest.isRefresh() || destinationIndices.isEmpty()) { finishHim(null, indexingFailures, searchFailures, timedOut); return; } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index daf9a15daea..06f5226a7f4 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -458,23 +458,29 @@ public class AsyncBulkByScrollActionTests extends ESTestCase { } public void testRefreshIsFalseByDefault() throws Exception { - refreshTestCase(null, false); + refreshTestCase(null, true, false); } - public void testRefreshFalseDoesntMakeVisible() throws Exception { - refreshTestCase(false, false); + public void testRefreshFalseDoesntExecuteRefresh() throws Exception { + refreshTestCase(false, true, false); } - public void testRefreshTrueMakesVisible() throws Exception { - refreshTestCase(true, true); + public void testRefreshTrueExecutesRefresh() throws Exception { + refreshTestCase(true, true, true); } - private void refreshTestCase(Boolean refresh, boolean shouldRefresh) { + public void testRefreshTrueSkipsRefreshIfNoDestinationIndexes() throws Exception { + refreshTestCase(true, false, false); + } + + private void refreshTestCase(Boolean refresh, boolean addDestinationIndexes, boolean shouldRefresh) { if (refresh != null) { mainRequest.setRefresh(refresh); } DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction(); - action.addDestinationIndices(singleton("foo")); + if (addDestinationIndexes) { + action.addDestinationIndices(singleton("foo")); + } action.startNormalTermination(emptyList(), emptyList(), false); if (shouldRefresh) { assertArrayEquals(new String[] {"foo"}, client.lastRefreshRequest.get().indices()); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java index a8bf3ea2959..650d71f62ad 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java @@ -36,7 +36,6 @@ import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.BlobStoreException; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import java.util.ArrayList; @@ -47,8 +46,6 @@ import java.util.Locale; */ public class S3BlobStore extends AbstractComponent implements BlobStore { - public static final ByteSizeValue MIN_BUFFER_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB); - private final AmazonS3 client; private final String bucket; @@ -72,12 +69,7 @@ public class S3BlobStore extends AbstractComponent implements BlobStore { this.bucket = bucket; this.region = region; this.serverSideEncryption = serverSideEncryption; - - this.bufferSize = (bufferSize != null) ? bufferSize : MIN_BUFFER_SIZE; - if (this.bufferSize.getBytes() < MIN_BUFFER_SIZE.getBytes()) { - throw new BlobStoreException("Detected a buffer_size for the S3 storage lower than [" + MIN_BUFFER_SIZE + "]"); - } - + this.bufferSize = bufferSize; this.cannedACL = initCannedACL(cannedACL); this.numberOfRetries = maxRetries; this.storageClass = initStorageClass(storageClass); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index fde774a6b92..a09d57ebc93 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -106,19 +106,21 @@ public class S3Repository extends BlobStoreRepository { * repositories.s3.buffer_size: Minimum threshold below which the chunk is uploaded using a single request. Beyond this threshold, * the S3 repository will use the AWS Multipart Upload API to split the chunk into several parts, each of buffer_size length, and * to upload each part in its own request. Note that setting a buffer size lower than 5mb is not allowed since it will prevents the - * use of the Multipart API and may result in upload errors. Defaults to 5mb. + * use of the Multipart API and may result in upload errors. Defaults to 100m. */ Setting BUFFER_SIZE_SETTING = - Setting.byteSizeSetting("repositories.s3.buffer_size", S3BlobStore.MIN_BUFFER_SIZE, Property.NodeScope); + Setting.byteSizeSetting("repositories.s3.buffer_size", new ByteSizeValue(100, ByteSizeUnit.MB), + new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope); /** * repositories.s3.max_retries: Number of retries in case of S3 errors. Defaults to 3. */ Setting MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope); /** - * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 100m. + * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g. */ Setting CHUNK_SIZE_SETTING = - Setting.byteSizeSetting("repositories.s3.chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB), Property.NodeScope); + Setting.byteSizeSetting("repositories.s3.chunk_size", new ByteSizeValue(1, ByteSizeUnit.GB), + new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope); /** * repositories.s3.compress: When set to true metadata files are stored in compressed format. This setting doesn’t affect index * files that are already compressed by default. Defaults to false. @@ -187,7 +189,8 @@ public class S3Repository extends BlobStoreRepository { * @see Repositories#BUFFER_SIZE_SETTING */ Setting BUFFER_SIZE_SETTING = - Setting.byteSizeSetting("buffer_size", S3BlobStore.MIN_BUFFER_SIZE, Property.NodeScope); + Setting.byteSizeSetting("buffer_size", new ByteSizeValue(100, ByteSizeUnit.MB), + new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope); /** * max_retries * @see Repositories#MAX_RETRIES_SETTING @@ -197,7 +200,9 @@ public class S3Repository extends BlobStoreRepository { * chunk_size * @see Repositories#CHUNK_SIZE_SETTING */ - Setting CHUNK_SIZE_SETTING = Setting.byteSizeSetting("chunk_size", "-1", Property.NodeScope); + Setting CHUNK_SIZE_SETTING = + Setting.byteSizeSetting("chunk_size", new ByteSizeValue(1, ByteSizeUnit.GB), + new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.TB), Property.NodeScope); /** * compress * @see Repositories#COMPRESS_SETTING @@ -260,6 +265,12 @@ public class S3Repository extends BlobStoreRepository { this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING); this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING); + // We make sure that chunkSize is bigger or equal than/to bufferSize + if (this.chunkSize.getBytes() < bufferSize.getBytes()) { + throw new RepositoryException(name.name(), Repository.CHUNK_SIZE_SETTING.getKey() + " (" + this.chunkSize + + ") can't be lower than " + Repository.BUFFER_SIZE_SETTING.getKey() + " (" + bufferSize + ")."); + } + // Parse and validate the user's S3 Storage Class setting String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING); String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java index 7d881e0dd30..c5c617e8591 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java @@ -21,12 +21,20 @@ package org.elasticsearch.cloud.aws; import com.amazonaws.Protocol; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.RepositoryName; import org.elasticsearch.repositories.RepositorySettings; +import org.elasticsearch.repositories.s3.S3Repository; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; + import static org.elasticsearch.repositories.s3.S3Repository.Repositories; import static org.elasticsearch.repositories.s3.S3Repository.Repository; import static org.elasticsearch.repositories.s3.S3Repository.getValue; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isEmptyString; @@ -110,9 +118,9 @@ public class RepositoryS3SettingsTests extends ESTestCase { assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("global-signer")); assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING), is(false)); - assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(5L)); + assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(100L)); assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(3)); - assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getMb(), is(100L)); + assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getGb(), is(1L)); assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(false)); assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString()); @@ -138,9 +146,9 @@ public class RepositoryS3SettingsTests extends ESTestCase { assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("s3-signer")); assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING), is(false)); - assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(5L)); + assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(100L)); assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(3)); - assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getMb(), is(100L)); + assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getGb(), is(1L)); assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(false)); assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString()); assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString()); @@ -292,6 +300,25 @@ public class RepositoryS3SettingsTests extends ESTestCase { assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), is("repository-basepath")); } + /** + * We test wrong Chunk and Buffer settings + */ + public void testInvalidChunkBufferSizeRepositorySettings() throws IOException { + // chunk < buffer should fail + internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.MB), + "chunk_size (5mb) can't be lower than buffer_size (10mb)."); + // chunk > buffer should pass + internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(10, ByteSizeUnit.MB), null); + // chunk = buffer should pass + internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.MB), null); + // buffer < 5mb should fail + internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(4, ByteSizeUnit.MB), new ByteSizeValue(10, ByteSizeUnit.MB), + "Failed to parse value [4mb] for setting [buffer_size] must be >= 5mb"); + // chunk > 5tb should fail + internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(6, ByteSizeUnit.TB), + "Failed to parse value [6tb] for setting [chunk_size] must be =< 5tb"); + } + private Settings buildSettings(Settings... global) { Settings.Builder builder = Settings.builder(); for (Settings settings : global) { @@ -299,4 +326,28 @@ public class RepositoryS3SettingsTests extends ESTestCase { } return builder.build(); } + + private void internalTestInvalidChunkBufferSizeSettings(ByteSizeValue buffer, ByteSizeValue chunk, String expectedMessage) + throws IOException { + Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES); + RepositorySettings s3RepositorySettings = new RepositorySettings(nodeSettings, Settings.builder() + .put(Repository.BUFFER_SIZE_SETTING.getKey(), buffer) + .put(Repository.CHUNK_SIZE_SETTING.getKey(), chunk) + .build()); + + try { + new S3Repository(new RepositoryName("s3", "s3repo"), s3RepositorySettings, null, null); + fail("We should either raise a NPE or a RepositoryException or a IllegalArgumentException"); + } catch (RepositoryException e) { + assertThat(e.getDetailedMessage(), containsString(expectedMessage)); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString(expectedMessage)); + } catch (NullPointerException e) { + // Because we passed to the CTOR a Null AwsS3Service, we get a NPE which is expected + // in the context of this test + if (expectedMessage != null) { + fail("We should have raised a RepositoryException"); + } + } + } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java index f023b64211f..0518d657e23 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3OutputStreamTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.cloud.aws.blobstore; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.test.ESTestCase; import java.io.ByteArrayOutputStream; @@ -33,7 +35,7 @@ import static org.hamcrest.Matchers.is; * Unit test for {@link S3OutputStream}. */ public class S3OutputStreamTests extends ESTestCase { - private static final int BUFFER_SIZE = S3BlobStore.MIN_BUFFER_SIZE.bytesAsInt(); + private static final int BUFFER_SIZE = new ByteSizeValue(5, ByteSizeUnit.MB).bytesAsInt(); public void testWriteLessDataThanBufferSize() throws IOException { MockDefaultS3OutputStream out = newS3OutputStream(BUFFER_SIZE); diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle index 93d361c989c..c86b21dfea4 100644 --- a/qa/backwards-5.0/build.gradle +++ b/qa/backwards-5.0/build.gradle @@ -18,6 +18,6 @@ integTest { cluster { numNodes = 2 numBwcNodes = 1 - bwcVersion = "5.0.0-SNAPSHOT" // this is the same as the current version until we released the first RC + bwcVersion = "5.0.0-alpha1-SNAPSHOT" // this is the same as the current version until we released the first RC } } diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index fb69c817f3a..106385578e9 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -19,6 +19,22 @@ package org.elasticsearch.plugins; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.google.common.jimfs.Configuration; +import com.google.common.jimfs.Jimfs; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.SuppressForbidden; +import org.elasticsearch.Version; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.cli.UserError; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.io.PathUtilsForTesting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.PosixPermissionsResetter; +import org.junit.After; + import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; @@ -26,6 +42,7 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -36,46 +53,100 @@ import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.PosixFileAttributeView; import java.nio.file.attribute.PosixFileAttributes; import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.Version; -import org.elasticsearch.cli.MockTerminal; -import org.elasticsearch.cli.UserError; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.PosixPermissionsResetter; -import org.junit.BeforeClass; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.containsInAnyOrder; @LuceneTestCase.SuppressFileSystems("*") public class InstallPluginCommandTests extends ESTestCase { - private static boolean isPosix; + private final Function temp; - @BeforeClass - public static void checkPosix() throws IOException { - isPosix = Files.getFileAttributeView(createTempFile(), PosixFileAttributeView.class) != null; + private final FileSystem fs; + private final boolean isPosix; + private final boolean isReal; + private final String javaIoTmpdir; + + @SuppressForbidden(reason = "sets java.io.tmpdir") + public InstallPluginCommandTests(FileSystem fs, Function temp) { + this.fs = fs; + this.temp = temp; + this.isPosix = fs.supportedFileAttributeViews().contains("posix"); + this.isReal = fs == PathUtils.getDefaultFileSystem(); + PathUtilsForTesting.installMock(fs); + javaIoTmpdir = System.getProperty("java.io.tmpdir"); + System.setProperty("java.io.tmpdir", temp.apply("tmpdir").toString()); + } + + @After + @SuppressForbidden(reason = "resets java.io.tmpdir") + public void tearDown() throws Exception { + System.setProperty("java.io.tmpdir", javaIoTmpdir); + PathUtilsForTesting.teardown(); + super.tearDown(); + } + + @ParametersFactory + public static Iterable parameters() { + class Parameter { + private final FileSystem fileSystem; + private final Function temp; + + public Parameter(FileSystem fileSystem, String root) { + this(fileSystem, s -> { + try { + return Files.createTempDirectory(fileSystem.getPath(root), s); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + public Parameter(FileSystem fileSystem, Function temp) { + this.fileSystem = fileSystem; + this.temp = temp; + } + } + List parameters = new ArrayList<>(); + parameters.add(new Parameter(Jimfs.newFileSystem(Configuration.windows()), "c:\\")); + parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.osX())), "/")); + parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.unix())), "/")); + parameters.add(new Parameter(PathUtils.getDefaultFileSystem(), LuceneTestCase::createTempDir )); + return parameters.stream().map(p -> new Object[] { p.fileSystem, p.temp }).collect(Collectors.toList()); + } + + private static Configuration toPosix(Configuration configuration) { + return configuration.toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build(); } /** Creates a test environment with bin, config and plugins directories. */ - static Environment createEnv() throws IOException { - Path home = createTempDir(); + static Environment createEnv(FileSystem fs, Function temp) throws IOException { + Path home = temp.apply("install-plugin-command-tests"); Files.createDirectories(home.resolve("bin")); Files.createFile(home.resolve("bin").resolve("elasticsearch")); Files.createDirectories(home.resolve("config")); Files.createFile(home.resolve("config").resolve("elasticsearch.yml")); - Files.createDirectories(home.resolve("plugins")); + Path plugins = Files.createDirectories(home.resolve("plugins")); + assertTrue(Files.exists(plugins)); Settings settings = Settings.builder() .put("path.home", home) .build(); return new Environment(settings); } + static Path createPluginDir(Function temp) throws IOException { + return temp.apply("pluginDir"); + } + /** creates a fake jar file with empty class files */ static void writeJar(Path jar, String... classes) throws IOException { try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(jar))) { @@ -115,14 +186,40 @@ public class InstallPluginCommandTests extends ESTestCase { } static MockTerminal installPlugin(String pluginUrl, Environment env) throws Exception { + return installPlugin(pluginUrl, env, false); + } + + static MockTerminal installPlugin(String pluginUrl, Environment env, boolean jarHellCheck) throws Exception { MockTerminal terminal = new MockTerminal(); - new InstallPluginCommand(env).execute(terminal, pluginUrl, true); + new InstallPluginCommand(env) { + @Override + void jarHellCheck(Path candidate, Path pluginsDir) throws Exception { + if (jarHellCheck) { + super.jarHellCheck(candidate, pluginsDir); + } + } + }.execute(terminal, pluginUrl, true); return terminal; } void assertPlugin(String name, Path original, Environment env) throws IOException { Path got = env.pluginsFile().resolve(name); assertTrue("dir " + name + " exists", Files.exists(got)); + + if (isPosix) { + Set perms = Files.getPosixFilePermissions(got); + assertThat( + perms, + containsInAnyOrder( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE, + PosixFilePermission.GROUP_READ, + PosixFilePermission.GROUP_EXECUTE, + PosixFilePermission.OTHERS_READ, + PosixFilePermission.OTHERS_EXECUTE)); + } + assertTrue("jar was copied", Files.exists(got.resolve("plugin.jar"))); assertFalse("bin was not copied", Files.exists(got.resolve("bin"))); assertFalse("config was not copied", Files.exists(got.resolve("config"))); @@ -152,6 +249,16 @@ public class InstallPluginCommandTests extends ESTestCase { Path configDir = env.configFile().resolve(name); assertTrue("config dir exists", Files.exists(configDir)); assertTrue("config is a dir", Files.isDirectory(configDir)); + + if (isPosix) { + Path configRoot = env.configFile(); + PosixFileAttributes configAttributes = + Files.getFileAttributeView(configRoot, PosixFileAttributeView.class).readAttributes(); + PosixFileAttributes attributes = Files.getFileAttributeView(configDir, PosixFileAttributeView.class).readAttributes(); + assertThat(attributes.owner(), equalTo(configAttributes.owner())); + assertThat(attributes.group(), equalTo(configAttributes.group())); + } + try (DirectoryStream stream = Files.newDirectoryStream(configDir)) { for (Path file : stream) { assertFalse("not a dir", Files.isDirectory(file)); @@ -172,16 +279,16 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testSomethingWorks() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); String pluginZip = createPlugin("fake", pluginDir); installPlugin(pluginZip, env); assertPlugin("fake", pluginDir, env); } public void testSpaceInUrl() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); String pluginZip = createPlugin("fake", pluginDir); Path pluginZipWithSpaces = createTempFile("foo bar", ".zip"); try (InputStream in = new URL(pluginZip).openStream()) { @@ -192,28 +299,30 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testMalformedUrlNotMaven() throws Exception { + Environment env = createEnv(fs, temp); // has two colons, so it appears similar to maven coordinates MalformedURLException e = expectThrows(MalformedURLException.class, () -> { - installPlugin("://host:1234", createEnv()); + installPlugin("://host:1234", env); }); assertTrue(e.getMessage(), e.getMessage().contains("no protocol")); } public void testPluginsDirMissing() throws Exception { - Environment env = createEnv(); + Environment env = createEnv(fs, temp); Files.delete(env.pluginsFile()); - Path pluginDir = createTempDir(); + Path pluginDir = createPluginDir(temp); String pluginZip = createPlugin("fake", pluginDir); installPlugin(pluginZip, env); assertPlugin("fake", pluginDir, env); } public void testPluginsDirReadOnly() throws Exception { - assumeTrue("posix filesystem", isPosix); - Environment env = createEnv(); + assumeTrue("posix and filesystem", isPosix && isReal); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); try (PosixPermissionsResetter pluginsAttrs = new PosixPermissionsResetter(env.pluginsFile())) { pluginsAttrs.setPermissions(new HashSet<>()); - String pluginZip = createPlugin("fake", createTempDir()); + String pluginZip = createPlugin("fake", pluginDir); IOException e = expectThrows(IOException.class, () -> { installPlugin(pluginZip, env); }); @@ -223,8 +332,9 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testBuiltinModule() throws Exception { - Environment env = createEnv(); - String pluginZip = createPlugin("lang-groovy", createTempDir()); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); + String pluginZip = createPlugin("lang-groovy", pluginDir); UserError e = expectThrows(UserError.class, () -> { installPlugin(pluginZip, env); }); @@ -233,66 +343,36 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testJarHell() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); - writeJar(pluginDir.resolve("other.jar"), "FakePlugin"); - String pluginZip = createPlugin("fake", pluginDir); // adds plugin.jar with FakePlugin + // jar hell test needs a real filesystem + assumeTrue("real filesystem", isReal); + Environment environment = createEnv(fs, temp); + Path pluginDirectory = createPluginDir(temp); + writeJar(pluginDirectory.resolve("other.jar"), "FakePlugin"); + String pluginZip = createPlugin("fake", pluginDirectory); // adds plugin.jar with FakePlugin IllegalStateException e = expectThrows(IllegalStateException.class, () -> { - installPlugin(pluginZip, env); + installPlugin(pluginZip, environment, true); }); assertTrue(e.getMessage(), e.getMessage().contains("jar hell")); - assertInstallCleaned(env); + assertInstallCleaned(environment); } public void testIsolatedPlugins() throws Exception { - Environment env = createEnv(); + Environment env = createEnv(fs, temp); // these both share the same FakePlugin class - Path pluginDir1 = createTempDir(); + Path pluginDir1 = createPluginDir(temp); String pluginZip1 = createPlugin("fake1", pluginDir1); installPlugin(pluginZip1, env); - Path pluginDir2 = createTempDir(); + Path pluginDir2 = createPluginDir(temp); String pluginZip2 = createPlugin("fake2", pluginDir2); installPlugin(pluginZip2, env); assertPlugin("fake1", pluginDir1, env); assertPlugin("fake2", pluginDir2, env); } - public void testPurgatoryJarHell() throws Exception { - Environment env = createEnv(); - Path pluginDir1 = createTempDir(); - PluginTestUtil.writeProperties(pluginDir1, - "description", "fake desc", - "name", "fake1", - "version", "1.0", - "elasticsearch.version", Version.CURRENT.toString(), - "java.version", System.getProperty("java.specification.version"), - "classname", "FakePlugin", - "isolated", "false"); - writeJar(pluginDir1.resolve("plugin.jar"), "FakePlugin"); - String pluginZip1 = writeZip(pluginDir1, "elasticsearch"); - installPlugin(pluginZip1, env); - - Path pluginDir2 = createTempDir(); - PluginTestUtil.writeProperties(pluginDir2, - "description", "fake desc", - "name", "fake2", - "version", "1.0", - "elasticsearch.version", Version.CURRENT.toString(), - "java.version", System.getProperty("java.specification.version"), - "classname", "FakePlugin", - "isolated", "false"); - writeJar(pluginDir2.resolve("plugin.jar"), "FakePlugin"); - String pluginZip2 = writeZip(pluginDir2, "elasticsearch"); - IllegalStateException e = expectThrows(IllegalStateException.class, () -> { - installPlugin(pluginZip2, env); - }); - assertTrue(e.getMessage(), e.getMessage().contains("jar hell")); - assertInstallCleaned(env); - } - public void testExistingPlugin() throws Exception { - Environment env = createEnv(); - String pluginZip = createPlugin("fake", createTempDir()); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); + String pluginZip = createPlugin("fake", pluginDir); installPlugin(pluginZip, env); UserError e = expectThrows(UserError.class, () -> { installPlugin(pluginZip, env); @@ -302,8 +382,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testBin() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path binDir = pluginDir.resolve("bin"); Files.createDirectory(binDir); Files.createFile(binDir.resolve("somescript")); @@ -313,8 +393,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testBinNotDir() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path binDir = pluginDir.resolve("bin"); Files.createFile(binDir); String pluginZip = createPlugin("fake", pluginDir); @@ -326,8 +406,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testBinContainsDir() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path dirInBinDir = pluginDir.resolve("bin").resolve("foo"); Files.createDirectories(dirInBinDir); Files.createFile(dirInBinDir.resolve("somescript")); @@ -340,8 +420,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testBinConflict() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path binDir = pluginDir.resolve("bin"); Files.createDirectory(binDir); Files.createFile(binDir.resolve("somescript")); @@ -355,8 +435,8 @@ public class InstallPluginCommandTests extends ESTestCase { public void testBinPermissions() throws Exception { assumeTrue("posix filesystem", isPosix); - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path binDir = pluginDir.resolve("bin"); Files.createDirectory(binDir); Files.createFile(binDir.resolve("somescript")); @@ -372,8 +452,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testConfig() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path configDir = pluginDir.resolve("config"); Files.createDirectory(configDir); Files.createFile(configDir.resolve("custom.yaml")); @@ -383,11 +463,11 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testExistingConfig() throws Exception { - Environment env = createEnv(); + Environment env = createEnv(fs, temp); Path envConfigDir = env.configFile().resolve("fake"); Files.createDirectories(envConfigDir); Files.write(envConfigDir.resolve("custom.yaml"), "existing config".getBytes(StandardCharsets.UTF_8)); - Path pluginDir = createTempDir(); + Path pluginDir = createPluginDir(temp); Path configDir = pluginDir.resolve("config"); Files.createDirectory(configDir); Files.write(configDir.resolve("custom.yaml"), "new config".getBytes(StandardCharsets.UTF_8)); @@ -402,8 +482,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testConfigNotDir() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path configDir = pluginDir.resolve("config"); Files.createFile(configDir); String pluginZip = createPlugin("fake", pluginDir); @@ -415,8 +495,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testConfigContainsDir() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path dirInConfigDir = pluginDir.resolve("config").resolve("foo"); Files.createDirectories(dirInConfigDir); Files.createFile(dirInConfigDir.resolve("myconfig.yml")); @@ -429,8 +509,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testConfigConflict() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Path configDir = pluginDir.resolve("config"); Files.createDirectory(configDir); Files.createFile(configDir.resolve("myconfig.yml")); @@ -443,8 +523,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testMissingDescriptor() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Files.createFile(pluginDir.resolve("fake.yml")); String pluginZip = writeZip(pluginDir, "elasticsearch"); NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> { @@ -455,8 +535,8 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testMissingDirectory() throws Exception { - Environment env = createEnv(); - Path pluginDir = createTempDir(); + Environment env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); Files.createFile(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES)); String pluginZip = writeZip(pluginDir, null); UserError e = expectThrows(UserError.class, () -> { @@ -467,13 +547,14 @@ public class InstallPluginCommandTests extends ESTestCase { } public void testZipRelativeOutsideEntryName() throws Exception { + Environment env = createEnv(fs, temp); Path zip = createTempDir().resolve("broken.zip"); try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) { stream.putNextEntry(new ZipEntry("elasticsearch/../blah")); } String pluginZip = zip.toUri().toURL().toString(); IOException e = expectThrows(IOException.class, () -> { - installPlugin(pluginZip, createEnv()); + installPlugin(pluginZip, env); }); assertTrue(e.getMessage(), e.getMessage().contains("resolving outside of plugin directory")); } diff --git a/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats b/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats index 65d212e56d8..deeb37d54e5 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats +++ b/qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats @@ -42,6 +42,7 @@ setup() { @test "[UPGRADE] install old version" { clean_before_test install_package -v $(cat upgrade_from_version) + perl -p -i -e 's/es.logger.level: INFO/es.logger.level: DEBUG/' /etc/elasticsearch/logging.yml } @test "[UPGRADE] start old version" { @@ -80,10 +81,12 @@ setup() { @test "[UPGRADE] install version under test" { install_package -u + perl -p -i -e 's/es.logger.level: INFO/es.logger.level: DEBUG/' /etc/elasticsearch/logging.yml } @test "[UPGRADE] start version under test" { - start_elasticsearch_service yellow + start_elasticsearch_service yellow library + wait_for_elasticsearch_status yellow library2 } @test "[UPGRADE] check elasticsearch version is version under test" { diff --git a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash index cb18363d60a..c8c1d6870a5 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash @@ -270,10 +270,11 @@ clean_before_test() { # $1 - expected status - defaults to green start_elasticsearch_service() { local desiredStatus=${1:-green} + local index=$2 run_elasticsearch_service 0 - wait_for_elasticsearch_status $desiredStatus + wait_for_elasticsearch_status $desiredStatus $index if [ -r "/tmp/elasticsearch/elasticsearch.pid" ]; then pid=$(cat /tmp/elasticsearch/elasticsearch.pid) @@ -382,9 +383,14 @@ stop_elasticsearch_service() { # $1 - expected status - defaults to green wait_for_elasticsearch_status() { local desiredStatus=${1:-green} + local index=$2 + + if [ -f /var/log/elasitcsearch/elasticsearch.log ]; then + cat /var/log/elasticsearch/elasticsearch.log >&3 + fi echo "Making sure elasticsearch is up..." - wget -O - --retry-connrefused --waitretry=1 --timeout=60 --tries 60 http://localhost:9200 || { + wget -O - --retry-connrefused --waitretry=1 --timeout=60 --tries 60 http://localhost:9200/_cluster/health || { echo "Looks like elasticsearch never started. Here is its log:" if [ -e "$ESLOG/elasticsearch.log" ]; then cat "$ESLOG/elasticsearch.log" @@ -392,15 +398,22 @@ wait_for_elasticsearch_status() { echo "The elasticsearch log doesn't exist. Maybe /var/log/messages has something:" tail -n20 /var/log/messages fi + cat /var/log/elasticsearch/elasticsearch.log >&3 false } - echo "Tring to connect to elasticsearch and wait for expected status..." - curl -sS "http://localhost:9200/_cluster/health?wait_for_status=$desiredStatus&timeout=60s&pretty" + if [ -z "index" ]; then + echo "Tring to connect to elasticsearch and wait for expected status $desiredStatus..." + curl -sS "http://localhost:9200/_cluster/health?wait_for_status=$desiredStatus&timeout=60s&pretty" + else + echo "Trying to connect to elasticsearch and wait for expected status $desiredStatus for index $index" + curl -sS "http://localhost:9200/_cluster/health/$index?wait_for_status=$desiredStatus&timeout=60s&pretty" + fi if [ $? -eq 0 ]; then echo "Connected" else echo "Unable to connect to Elastisearch" + cat /var/log/elasticsearch/elasticsearch.log >&3 false fi diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml index 423ca29a002..770c5c8b441 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml @@ -49,31 +49,6 @@ - \s+ $/ ---- -"Simple alias with yaml body through format argument": - - - do: - indices.create: - index: test - - - do: - indices.put_alias: - index: test - name: test_alias - - - do: - cat.aliases: - format: yaml - - - match: - $body: | - /^---\n - -\s+alias:\s+"test_alias"\s+ - index:\s+"test"\s+ - filter:\s+"-"\s+ - routing.index:\s+"-"\s+ - routing.search:\s+"-"\s+$/ - --- "Complex alias": diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml new file mode 100644 index 00000000000..c892891f08f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml @@ -0,0 +1,29 @@ +--- +"Simple alias with yaml body through format argument": + + - skip: + features: yaml + + - do: + indices.create: + index: test + + - do: + indices.put_alias: + index: test + name: test_alias + + - do: + cat.aliases: + format: yaml + + - match: + $body: | + /^---\n + -\s+alias:\s+"test_alias"\s+ + index:\s+"test"\s+ + filter:\s+"-"\s+ + routing.index:\s+"-"\s+ + routing.search:\s+"-"\s+$/ + + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml index 780edacd7b6..233e21bf470 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yaml @@ -138,16 +138,6 @@ /^ id \s+ snapshot.type \s+ snapshot.active \s+ snapshot.size \s+ snapshot.queue \s+ snapshot.queueSize \s+ snapshot.rejected \s+ snapshot.largest \s+ snapshot.completed \s+ snapshot.min \s+ snapshot.max \s+ snapshot.keepAlive \n (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - - do: - cat.thread_pool: - h: id,suggest.type,suggest.active,suggest.size,suggest.queue,suggest.queueSize,suggest.rejected,suggest.largest,suggest.completed,suggest.min,suggest.max,suggest.keepAlive - v: true - - - match: - $body: | - /^ id \s+ suggest.type \s+ suggest.active \s+ suggest.size \s+ suggest.queue \s+ suggest.queueSize \s+ suggest.rejected \s+ suggest.largest \s+ suggest.completed \s+ suggest.min \s+ suggest.max \s+ suggest.keepAlive \n - (\S+ \s+ (cached|fixed|scaling)? \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d+ \s+ \d+ \s+ \d+ \s+ \d* \s+ \d* \s+ \S* \n)+ $/ - - do: cat.thread_pool: h: id,warmer.type,warmer.active,warmer.size,warmer.queue,warmer.queueSize,warmer.rejected,warmer.largest,warmer.completed,warmer.min,warmer.max,warmer.keepAlive diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml index 19598c7363e..7b88ac57080 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/11_metric.yaml @@ -34,7 +34,6 @@ setup: - is_true: _all.total.completion - is_true: _all.total.segments - is_true: _all.total.translog - - is_true: _all.total.suggest - is_true: _all.total.recovery --- @@ -56,7 +55,6 @@ setup: - is_true: _all.total.completion - is_true: _all.total.segments - is_true: _all.total.translog - - is_true: _all.total.suggest - is_true: _all.total.recovery --- @@ -78,7 +76,6 @@ setup: - is_false: _all.total.completion - is_false: _all.total.segments - is_false: _all.total.translog - - is_false: _all.total.suggest - is_false: _all.total.recovery --- @@ -100,7 +97,6 @@ setup: - is_false: _all.total.completion - is_false: _all.total.segments - is_false: _all.total.translog - - is_false: _all.total.suggest - is_false: _all.total.recovery @@ -123,6 +119,5 @@ setup: - is_false: _all.total.completion - is_false: _all.total.segments - is_false: _all.total.translog - - is_false: _all.total.suggest - is_true: _all.total.recovery diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java index 0f51f72e8e5..66ba1528b90 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java @@ -34,7 +34,7 @@ import java.util.List; */ public final class Features { - private static final List SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers"); + private static final List SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers", "yaml"); private Features() { diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java index ea1929a55b0..97683b620f0 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/VersionUtilsTests.java @@ -46,14 +46,14 @@ public class VersionUtilsTests extends ESTestCase { assertTrue(got.onOrBefore(Version.CURRENT)); // sub range - got = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0); + got = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_5_0_0_alpha1); assertTrue(got.onOrAfter(Version.V_2_0_0)); - assertTrue(got.onOrBefore(Version.V_5_0_0)); + assertTrue(got.onOrBefore(Version.V_5_0_0_alpha1)); // unbounded lower - got = VersionUtils.randomVersionBetween(random(), null, Version.V_5_0_0); + got = VersionUtils.randomVersionBetween(random(), null, Version.V_5_0_0_alpha1); assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); - assertTrue(got.onOrBefore(Version.V_5_0_0)); + assertTrue(got.onOrBefore(Version.V_5_0_0_alpha1)); got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.allVersions().get(0)); assertTrue(got.onOrAfter(VersionUtils.getFirstVersion())); assertTrue(got.onOrBefore(VersionUtils.allVersions().get(0))); @@ -71,8 +71,8 @@ public class VersionUtilsTests extends ESTestCase { assertEquals(got, VersionUtils.getFirstVersion()); got = VersionUtils.randomVersionBetween(random(), Version.CURRENT, Version.CURRENT); assertEquals(got, Version.CURRENT); - got = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.V_5_0_0); - assertEquals(got, Version.V_5_0_0); + got = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha1, Version.V_5_0_0_alpha1); + assertEquals(got, Version.V_5_0_0_alpha1); // implicit range of one got = VersionUtils.randomVersionBetween(random(), null, VersionUtils.getFirstVersion());