diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java index 58457417fa7..8fea01e6fab 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexService.java +++ b/core/src/main/java/org/elasticsearch/index/IndexService.java @@ -89,6 +89,7 @@ import java.util.Set; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.LongSupplier; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; @@ -448,13 +449,13 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust * Creates a new QueryShardContext. The context has not types set yet, if types are required set them via * {@link QueryShardContext#setTypes(String...)} */ - public QueryShardContext newQueryShardContext(IndexReader indexReader) { + public QueryShardContext newQueryShardContext(IndexReader indexReader, LongSupplier nowInMillis) { return new QueryShardContext( indexSettings, indexCache.bitsetFilterCache(), indexFieldData, mapperService(), similarityService(), nodeServicesProvider.getScriptService(), nodeServicesProvider.getIndicesQueriesRegistry(), nodeServicesProvider.getClient(), indexReader, - nodeServicesProvider.getClusterService().state() - ); + nodeServicesProvider.getClusterService().state(), + nowInMillis); } /** @@ -463,7 +464,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust * used for rewriting since it does not know about the current {@link IndexReader}. */ public QueryShardContext newQueryShardContext() { - return newQueryShardContext(null); + return newQueryShardContext(null, threadPool::estimatedTimeInMillis); } public ThreadPool getThreadPool() { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 717f0361552..62b86c98a8c 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -366,7 +366,7 @@ public class DateFieldMapper extends FieldMapper { return () -> { final SearchContext context = SearchContext.current(); return context != null - ? context.nowInMillis() + ? context.getQueryShardContext().nowInMillis() : System.currentTimeMillis(); }; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyDateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyDateFieldMapper.java index 29689d06dff..5428dbe74f5 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/LegacyDateFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/LegacyDateFieldMapper.java @@ -480,7 +480,7 @@ public class LegacyDateFieldMapper extends LegacyNumberFieldMapper { public Long call() { final SearchContext context = SearchContext.current(); return context != null - ? context.nowInMillis() + ? context.getQueryShardContext().nowInMillis() : System.currentTimeMillis(); } }; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TTLFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TTLFieldMapper.java index bb2da75dad0..ea11dc04ee4 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/TTLFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/TTLFieldMapper.java @@ -143,7 +143,7 @@ public class TTLFieldMapper extends MetadataFieldMapper { long now; SearchContext searchContext = SearchContext.current(); if (searchContext != null) { - now = searchContext.nowInMillis(); + now = searchContext.getQueryShardContext().nowInMillis(); } else { now = System.currentTimeMillis(); } diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index 454c808a5d5..f2c83ce2a73 100644 --- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -575,8 +575,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl } if (scriptFields != null) { for (ScriptField field : scriptFields) { - SearchScript searchScript = innerHitsContext.scriptService().search(innerHitsContext.lookup(), field.script(), - ScriptContext.Standard.SEARCH, Collections.emptyMap()); + SearchScript searchScript = innerHitsContext.getQueryShardContext().getSearchScript(field.script(), + ScriptContext.Standard.SEARCH, Collections.emptyMap()); innerHitsContext.scriptFields().add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField( field.fieldName(), searchScript, field.ignoreFailure())); } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index f12605088e6..d7637f46370 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -19,17 +19,24 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.client.Client; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptSettings; +import java.util.Collections; + /** * Context object used to rewrite {@link QueryBuilder} instances into simplified version. */ @@ -69,13 +76,6 @@ public class QueryRewriteContext implements ParseFieldMatcherSupplier { return indexSettings; } - /** - * Returns a script service to fetch scripts. - */ - public final ScriptService getScriptService() { - return scriptService; - } - /** * Return the MapperService. */ @@ -116,4 +116,12 @@ public class QueryRewriteContext implements ParseFieldMatcherSupplier { String defaultScriptLanguage = ScriptSettings.getLegacyDefaultLang(indexSettings.getNodeSettings()); return new QueryParseContext(defaultScriptLanguage, indicesQueriesRegistry, parser, indexSettings.getParseFieldMatcher()); } + + public BytesReference getTemplateBytes(Script template) { + ExecutableScript executable = scriptService.executable(template, + ScriptContext.Standard.SEARCH, Collections.emptyMap()); + return (BytesReference) executable.run(); + } + + } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 377afca2f68..11af6f919ae 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -26,6 +26,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.function.Function; +import java.util.function.LongSupplier; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexReader; import org.apache.lucene.queryparser.classic.MapperQueryParser; @@ -33,11 +36,13 @@ import org.apache.lucene.queryparser.classic.QueryParserSettings; import org.apache.lucene.search.Query; import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -54,7 +59,12 @@ import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.support.NestedScope; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SearchLookup; @@ -69,6 +79,8 @@ public class QueryShardContext extends QueryRewriteContext { private final IndexFieldDataService indexFieldDataService; private final IndexSettings indexSettings; private String[] types = Strings.EMPTY_ARRAY; + private boolean cachable = true; + private final SetOnce frozen = new SetOnce<>(); public void setTypes(String... types) { this.types = types; @@ -85,11 +97,12 @@ public class QueryShardContext extends QueryRewriteContext { private boolean mapUnmappedFieldAsString; private NestedScope nestedScope; private boolean isFilter; + private final LongSupplier nowInMillis; public QueryShardContext(IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, IndexFieldDataService indexFieldDataService, MapperService mapperService, SimilarityService similarityService, ScriptService scriptService, final IndicesQueriesRegistry indicesQueriesRegistry, Client client, - IndexReader reader, ClusterState clusterState) { + IndexReader reader, ClusterState clusterState, LongSupplier nowInMillis) { super(indexSettings, mapperService, scriptService, indicesQueriesRegistry, client, reader, clusterState); this.indexSettings = indexSettings; this.similarityService = similarityService; @@ -99,12 +112,13 @@ public class QueryShardContext extends QueryRewriteContext { this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields(); this.indicesQueriesRegistry = indicesQueriesRegistry; this.nestedScope = new NestedScope(); + this.nowInMillis = nowInMillis; } public QueryShardContext(QueryShardContext source) { this(source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.indicesQueriesRegistry, source.client, - source.reader, source.clusterState); + source.reader, source.clusterState, source.nowInMillis); this.types = source.getTypes(); } @@ -261,11 +275,8 @@ public class QueryShardContext extends QueryRewriteContext { } public long nowInMillis() { - SearchContext current = SearchContext.current(); - if (current != null) { - return current.nowInMillis(); - } - return System.currentTimeMillis(); + failIfFrozen(); + return nowInMillis.getAsLong(); } public NestedScope nestedScope() { @@ -327,4 +338,77 @@ public class QueryShardContext extends QueryRewriteContext { public final Index index() { return indexSettings.getIndex(); } + + /** + * Compiles (or retrieves from cache) and binds the parameters to the + * provided script + */ + public SearchScript getSearchScript(Script script, ScriptContext context, Map params) { + failIfFrozen(); + return scriptService.search(lookup(), script, context, params); + } + /** + * Returns a lazily created {@link SearchScript} that is compiled immediately but can be pulled later once all + * parameters are available. + */ + public Function, SearchScript> getLazySearchScript(Script script, ScriptContext context, + Map params) { + failIfFrozen(); + CompiledScript compile = scriptService.compile(script, context, params); + return (p) -> scriptService.search(lookup(), compile, p); + } + + /** + * Compiles (or retrieves from cache) and binds the parameters to the + * provided script + */ + public ExecutableScript getExecutableScript(Script script, ScriptContext context, Map params) { + failIfFrozen(); + return scriptService.executable(script, context, params); + } + + /** + * Returns a lazily created {@link ExecutableScript} that is compiled immediately but can be pulled later once all + * parameters are available. + */ + public Function, ExecutableScript> getLazyExecutableScript(Script script, ScriptContext context, + Map params) { + failIfFrozen(); + CompiledScript executable = scriptService.compile(script, context, params); + return (p) -> scriptService.executable(executable, p); + } + + /** + * if this method is called the query context will throw exception if methods are accessed + * that could yield different results across executions like {@link #getTemplateBytes(Script)} + */ + public void freezeContext() { + this.frozen.set(Boolean.TRUE); + } + + /** + * This method fails if {@link #freezeContext()} is called before on this context. + * This is used to seal + */ + protected void failIfFrozen() { + this.cachable = false; + if (frozen.get() == Boolean.TRUE) { + throw new IllegalArgumentException("features that prevent cachability are disabled on this context"); + } else { + assert frozen.get() == null : frozen.get(); + } + } + + @Override + public BytesReference getTemplateBytes(Script template) { + failIfFrozen(); + return super.getTemplateBytes(template); + } + + /** + * Returns true iff the result of the processed search request is cachable. Otherwise false + */ + public boolean isCachable() { + return cachable; + } } diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index 3ff924b28db..4501d12b91f 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -41,7 +41,6 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; -import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -134,7 +133,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder @Override protected Query doToQuery(QueryShardContext context) throws IOException { - return new ScriptQuery(script, context.getScriptService(), context.lookup()); + return new ScriptQuery(script, context.getSearchScript(script, ScriptContext.Standard.SEARCH, Collections.emptyMap())); } static class ScriptQuery extends Query { @@ -143,9 +142,9 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder private final SearchScript searchScript; - public ScriptQuery(Script script, ScriptService scriptService, SearchLookup searchLookup) { + public ScriptQuery(Script script, SearchScript searchScript) { this.script = script; - this.searchScript = scriptService.search(searchLookup, script, ScriptContext.Standard.SEARCH, Collections.emptyMap()); + this.searchScript = searchScript; } @Override @@ -216,4 +215,6 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder protected boolean doEquals(ScriptQueryBuilder other) { return Objects.equals(script, other.script); } + + } diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java index 4f4a3b54a21..f630c0c9b76 100644 --- a/core/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java @@ -49,12 +49,19 @@ public class RandomScoreFunctionBuilder extends ScoreFunctionBuilder totalCompilesPerMinute) { @@ -488,7 +488,15 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust */ public SearchScript search(SearchLookup lookup, Script script, ScriptContext scriptContext, Map params) { CompiledScript compiledScript = compile(script, scriptContext, params); - return getScriptEngineServiceForLang(compiledScript.lang()).search(compiledScript, lookup, script.getParams()); + return search(lookup, compiledScript, script.getParams()); + } + + /** + * Binds provided parameters to a compiled script returning a + * {@link SearchScript} ready for execution + */ + public SearchScript search(SearchLookup lookup, CompiledScript compiledScript, Map params) { + return getScriptEngineServiceForLang(compiledScript.lang()).search(compiledScript, lookup, params); } private boolean isAnyScriptContextEnabled(String lang, ScriptType scriptType) { diff --git a/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 9aab786aa34..f8299693dcf 100644 --- a/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -51,7 +51,6 @@ import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.SearchContextAggregations; import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.fetch.FetchPhase; @@ -89,7 +88,6 @@ final class DefaultSearchContext extends SearchContext { private final Counter timeEstimateCounter; private SearchType searchType; private final Engine.Searcher engineSearcher; - private final ScriptService scriptService; private final BigArrays bigArrays; private final IndexShard indexShard; private final IndexService indexService; @@ -150,9 +148,9 @@ final class DefaultSearchContext extends SearchContext { private FetchPhase fetchPhase; DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget, Engine.Searcher engineSearcher, - IndexService indexService, IndexShard indexShard, ScriptService scriptService, - BigArrays bigArrays, Counter timeEstimateCounter, ParseFieldMatcher parseFieldMatcher, TimeValue timeout, - FetchPhase fetchPhase) { + IndexService indexService, IndexShard indexShard, + BigArrays bigArrays, Counter timeEstimateCounter, ParseFieldMatcher parseFieldMatcher, TimeValue timeout, + FetchPhase fetchPhase) { super(parseFieldMatcher); this.id = id; this.request = request; @@ -160,7 +158,6 @@ final class DefaultSearchContext extends SearchContext { this.searchType = request.searchType(); this.shardTarget = shardTarget; this.engineSearcher = engineSearcher; - this.scriptService = scriptService; // SearchContexts use a BigArrays that can circuit break this.bigArrays = bigArrays.withCircuitBreaking(); this.dfsResult = new DfsSearchResult(id, shardTarget); @@ -171,10 +168,17 @@ final class DefaultSearchContext extends SearchContext { this.searcher = new ContextIndexSearcher(engineSearcher, indexService.cache().query(), indexShard.getQueryCachingPolicy()); this.timeEstimateCounter = timeEstimateCounter; this.timeout = timeout; - queryShardContext = indexService.newQueryShardContext(searcher.getIndexReader()); + queryShardContext = indexService.newQueryShardContext(searcher.getIndexReader(), request::nowInMillis); queryShardContext.setTypes(request.types()); } + DefaultSearchContext(DefaultSearchContext source) { + this(source.id(), source.request(), source.shardTarget(), source.engineSearcher, source.indexService, source.indexShard(), + source.bigArrays(), source.timeEstimateCounter(), source.parseFieldMatcher(), source.timeout(), source.fetchPhase()); + } + + + @Override public void doClose() { // clear and scope phase we have @@ -358,11 +362,6 @@ final class DefaultSearchContext extends SearchContext { return originNanoTime; } - @Override - protected long nowInMillisImpl() { - return request.nowInMillis(); - } - @Override public ScrollContext scrollContext() { return this.scrollContext; @@ -501,11 +500,6 @@ final class DefaultSearchContext extends SearchContext { return indexService.similarityService(); } - @Override - public ScriptService scriptService() { - return scriptService; - } - @Override public BigArrays bigArrays() { return bigArrays; diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index 4334c5cf541..c3fbd123fc8 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -231,6 +231,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv */ private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final SearchContext context) throws Exception { final boolean canCache = indicesService.canCache(request, context); + context.getQueryShardContext().freezeContext(); if (canCache) { indicesService.loadIntoContext(request, context, queryPhase); } else { @@ -516,17 +517,18 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv } final SearchContext createContext(ShardSearchRequest request, @Nullable Engine.Searcher searcher) throws IOException { - - DefaultSearchContext context = createSearchContext(request, defaultSearchTimeout, searcher); - SearchContext.setCurrent(context); + final DefaultSearchContext context = createSearchContext(request, defaultSearchTimeout, searcher); try { - request.rewrite(context.getQueryShardContext()); - // reset that we have used nowInMillis from the context since it may - // have been rewritten so its no longer in the query and the request can - // be cached. If it is still present in the request (e.g. in a range - // aggregation) it will still be caught when the aggregation is - // evaluated. - context.resetNowInMillisUsed(); + // we clone the search context here just for rewriting otherwise we + // might end up with incorrect state since we are using now() or script services + // during rewrite and normalized / evaluate templates etc. + // NOTE this context doesn't need to be closed - the outer context will + // take care of this. + DefaultSearchContext rewriteContext = new DefaultSearchContext(context); + SearchContext.setCurrent(rewriteContext); + request.rewrite(rewriteContext.getQueryShardContext()); + SearchContext.setCurrent(context); + assert context.getQueryShardContext().isCachable(); if (request.scroll() != null) { context.scrollContext(new ScrollContext()); context.scrollContext().scroll = request.scroll(); @@ -568,7 +570,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv return new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget, engineSearcher, indexService, - indexShard, scriptService, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher, + indexShard, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher, timeout, fetchPhase); } @@ -735,7 +737,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv } if (source.scriptFields() != null) { for (org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField field : source.scriptFields()) { - SearchScript searchScript = context.scriptService().search(context.lookup(), field.script(), ScriptContext.Standard.SEARCH, + SearchScript searchScript = scriptService.search(context.lookup(), field.script(), ScriptContext.Standard.SEARCH, Collections.emptyMap()); context.scriptFields().add(new ScriptField(field.fieldName(), searchScript, field.ignoreFailure())); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java index d3b8857ccab..b20cf1b346a 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java @@ -21,6 +21,8 @@ package org.elasticsearch.search.aggregations.bucket.histogram; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.rounding.DateTimeUnit; +import org.elasticsearch.common.rounding.Rounding; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; @@ -35,8 +37,12 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; +import static java.util.Collections.unmodifiableMap; + /** * A builder for histograms on date fields. */ @@ -44,6 +50,29 @@ public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = InternalDateHistogram.TYPE.name(); + public static final Map DATE_FIELD_UNITS; + + static { + Map dateFieldUnits = new HashMap<>(); + dateFieldUnits.put("year", DateTimeUnit.YEAR_OF_CENTURY); + dateFieldUnits.put("1y", DateTimeUnit.YEAR_OF_CENTURY); + dateFieldUnits.put("quarter", DateTimeUnit.QUARTER); + dateFieldUnits.put("1q", DateTimeUnit.QUARTER); + dateFieldUnits.put("month", DateTimeUnit.MONTH_OF_YEAR); + dateFieldUnits.put("1M", DateTimeUnit.MONTH_OF_YEAR); + dateFieldUnits.put("week", DateTimeUnit.WEEK_OF_WEEKYEAR); + dateFieldUnits.put("1w", DateTimeUnit.WEEK_OF_WEEKYEAR); + dateFieldUnits.put("day", DateTimeUnit.DAY_OF_MONTH); + dateFieldUnits.put("1d", DateTimeUnit.DAY_OF_MONTH); + dateFieldUnits.put("hour", DateTimeUnit.HOUR_OF_DAY); + dateFieldUnits.put("1h", DateTimeUnit.HOUR_OF_DAY); + dateFieldUnits.put("minute", DateTimeUnit.MINUTES_OF_HOUR); + dateFieldUnits.put("1m", DateTimeUnit.MINUTES_OF_HOUR); + dateFieldUnits.put("second", DateTimeUnit.SECOND_OF_MINUTE); + dateFieldUnits.put("1s", DateTimeUnit.SECOND_OF_MINUTE); + DATE_FIELD_UNITS = unmodifiableMap(dateFieldUnits); + } + private long interval; private DateHistogramInterval dateHistogramInterval; private long offset = 0; @@ -245,8 +274,36 @@ public class DateHistogramAggregationBuilder @Override protected ValuesSourceAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { + Rounding rounding = createRounding(); + ExtendedBounds roundedBounds = null; + if (this.extendedBounds != null) { + // parse any string bounds to longs and round + roundedBounds = this.extendedBounds.parseAndValidate(name, context.searchContext(), config.format()).round(rounding); + } return new DateHistogramAggregatorFactory(name, type, config, interval, dateHistogramInterval, offset, order, keyed, minDocCount, - extendedBounds, context, parent, subFactoriesBuilder, metaData); + rounding, roundedBounds, context, parent, subFactoriesBuilder, metaData); + } + + private Rounding createRounding() { + Rounding.Builder tzRoundingBuilder; + if (dateHistogramInterval != null) { + DateTimeUnit dateTimeUnit = DATE_FIELD_UNITS.get(dateHistogramInterval.toString()); + if (dateTimeUnit != null) { + tzRoundingBuilder = Rounding.builder(dateTimeUnit); + } else { + // the interval is a time value? + tzRoundingBuilder = Rounding.builder( + TimeValue.parseTimeValue(dateHistogramInterval.toString(), null, getClass().getSimpleName() + ".interval")); + } + } else { + // the interval is an integer time value in millis? + tzRoundingBuilder = Rounding.builder(TimeValue.timeValueMillis(interval)); + } + if (timeZone() != null) { + tzRoundingBuilder.timeZone(timeZone()); + } + Rounding rounding = tzRoundingBuilder.build(); + return rounding; } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java index 79f81e28374..f1c4a6b4fae 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java @@ -19,9 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.histogram; -import org.elasticsearch.common.rounding.DateTimeUnit; import org.elasticsearch.common.rounding.Rounding; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -30,12 +28,9 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; -import static java.util.Collections.unmodifiableMap; - import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; @@ -44,29 +39,6 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; public final class DateHistogramAggregatorFactory extends ValuesSourceAggregatorFactory { - public static final Map DATE_FIELD_UNITS; - - static { - Map dateFieldUnits = new HashMap<>(); - dateFieldUnits.put("year", DateTimeUnit.YEAR_OF_CENTURY); - dateFieldUnits.put("1y", DateTimeUnit.YEAR_OF_CENTURY); - dateFieldUnits.put("quarter", DateTimeUnit.QUARTER); - dateFieldUnits.put("1q", DateTimeUnit.QUARTER); - dateFieldUnits.put("month", DateTimeUnit.MONTH_OF_YEAR); - dateFieldUnits.put("1M", DateTimeUnit.MONTH_OF_YEAR); - dateFieldUnits.put("week", DateTimeUnit.WEEK_OF_WEEKYEAR); - dateFieldUnits.put("1w", DateTimeUnit.WEEK_OF_WEEKYEAR); - dateFieldUnits.put("day", DateTimeUnit.DAY_OF_MONTH); - dateFieldUnits.put("1d", DateTimeUnit.DAY_OF_MONTH); - dateFieldUnits.put("hour", DateTimeUnit.HOUR_OF_DAY); - dateFieldUnits.put("1h", DateTimeUnit.HOUR_OF_DAY); - dateFieldUnits.put("minute", DateTimeUnit.MINUTES_OF_HOUR); - dateFieldUnits.put("1m", DateTimeUnit.MINUTES_OF_HOUR); - dateFieldUnits.put("second", DateTimeUnit.SECOND_OF_MINUTE); - dateFieldUnits.put("1s", DateTimeUnit.SECOND_OF_MINUTE); - DATE_FIELD_UNITS = unmodifiableMap(dateFieldUnits); - } - private final DateHistogramInterval dateHistogramInterval; private final long interval; private final long offset; @@ -74,10 +46,11 @@ public final class DateHistogramAggregatorFactory private final boolean keyed; private final long minDocCount; private final ExtendedBounds extendedBounds; + private Rounding rounding; public DateHistogramAggregatorFactory(String name, Type type, ValuesSourceConfig config, long interval, DateHistogramInterval dateHistogramInterval, long offset, InternalOrder order, boolean keyed, long minDocCount, - ExtendedBounds extendedBounds, AggregationContext context, AggregatorFactory parent, + Rounding rounding, ExtendedBounds extendedBounds, AggregationContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { super(name, type, config, context, parent, subFactoriesBuilder, metaData); this.interval = interval; @@ -87,34 +60,13 @@ public final class DateHistogramAggregatorFactory this.keyed = keyed; this.minDocCount = minDocCount; this.extendedBounds = extendedBounds; + this.rounding = rounding; } public long minDocCount() { return minDocCount; } - private Rounding createRounding() { - Rounding.Builder tzRoundingBuilder; - if (dateHistogramInterval != null) { - DateTimeUnit dateTimeUnit = DATE_FIELD_UNITS.get(dateHistogramInterval.toString()); - if (dateTimeUnit != null) { - tzRoundingBuilder = Rounding.builder(dateTimeUnit); - } else { - // the interval is a time value? - tzRoundingBuilder = Rounding.builder( - TimeValue.parseTimeValue(dateHistogramInterval.toString(), null, getClass().getSimpleName() + ".interval")); - } - } else { - // the interval is an integer time value in millis? - tzRoundingBuilder = Rounding.builder(TimeValue.timeValueMillis(interval)); - } - if (timeZone() != null) { - tzRoundingBuilder.timeZone(timeZone()); - } - Rounding rounding = tzRoundingBuilder.build(); - return rounding; - } - @Override protected Aggregator doCreateInternal(ValuesSource.Numeric valuesSource, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { @@ -126,18 +78,7 @@ public final class DateHistogramAggregatorFactory private Aggregator createAggregator(ValuesSource.Numeric valuesSource, Aggregator parent, List pipelineAggregators, Map metaData) throws IOException { - Rounding rounding = createRounding(); - // we need to round the bounds given by the user and we have to do it - // for every aggregator we create - // as the rounding is not necessarily an idempotent operation. - // todo we need to think of a better structure to the factory/agtor - // code so we won't need to do that - ExtendedBounds roundedBounds = null; - if (extendedBounds != null) { - // parse any string bounds to longs and round them - roundedBounds = extendedBounds.parseAndValidate(name, context.searchContext(), config.format()).round(rounding); - } - return new DateHistogramAggregator(name, factories, rounding, offset, order, keyed, minDocCount, roundedBounds, valuesSource, + return new DateHistogramAggregator(name, factories, rounding, offset, order, keyed, minDocCount, extendedBounds, valuesSource, config.format(), context, parent, pipelineAggregators, metaData); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java index 46fae19e49f..85160347612 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java @@ -153,11 +153,11 @@ public class ExtendedBounds implements ToXContent, Writeable { Long max = this.max; assert format != null; if (minAsStr != null) { - min = format.parseLong(minAsStr, false, context::nowInMillis); + min = format.parseLong(minAsStr, false, context.getQueryShardContext()::nowInMillis); } if (maxAsStr != null) { // TODO: Should we rather pass roundUp=true? - max = format.parseLong(maxAsStr, false, context::nowInMillis); + max = format.parseLong(maxAsStr, false, context.getQueryShardContext()::nowInMillis); } if (min != null && max != null && min.compareTo(max) > 0) { throw new SearchParseException(context, "[extended_bounds.min][" + min + "] cannot be greater than " + diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java index 27d826a644a..e35e4af9e78 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java @@ -118,10 +118,10 @@ public class RangeAggregator extends BucketsAggregator { Double from = this.from; Double to = this.to; if (fromAsStr != null) { - from = parser.parseDouble(fromAsStr, false, context::nowInMillis); + from = parser.parseDouble(fromAsStr, false, context.getQueryShardContext()::nowInMillis); } if (toAsStr != null) { - to = parser.parseDouble(toAsStr, false, context::nowInMillis); + to = parser.parseDouble(toAsStr, false, context.getQueryShardContext()::nowInMillis); } return new Range(key, from, fromAsStr, to, toAsStr); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java index 245297d72cc..46f5d881fa1 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java @@ -220,6 +220,7 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB @Override protected ValuesSourceAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { + this.significanceHeuristic.initialize(context.searchContext()); return new SignificantTermsAggregatorFactory(name, type, config, includeExclude, executionHint, filterBuilder, bucketCountThresholds, significanceHeuristic, context, parent, subFactoriesBuilder, metaData); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java index ab30e1b2d4a..1a5338f7458 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java @@ -91,7 +91,6 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac : searcher.count(filter); this.bucketCountThresholds = bucketCountThresholds; this.significanceHeuristic = significanceHeuristic; - this.significanceHeuristic.initialize(context.searchContext()); setFieldInfo(); } @@ -211,13 +210,13 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac } } assert execution != null; - + DocValueFormat format = config.format(); if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) { throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style include/exclude " + "settings as they can only be applied to string fields. Use an array of values for include/exclude clauses"); } - + return execution.create(name, factories, valuesSource, format, bucketCountThresholds, includeExclude, context, parent, significanceHeuristic, this, pipelineAggregators, metaData); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java index c933f9ef596..ee829943ae1 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java @@ -32,7 +32,6 @@ import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.Script.ScriptField; import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.support.XContentParseContext; import org.elasticsearch.search.internal.SearchContext; @@ -49,7 +48,7 @@ public class ScriptHeuristic extends SignificanceHeuristic { private final LongAccessor subsetDfHolder; private final LongAccessor supersetDfHolder; private final Script script; - ExecutableScript searchScript = null; + ExecutableScript executableScript = null; public ScriptHeuristic(Script script) { subsetSizeHolder = new LongAccessor(); @@ -73,20 +72,20 @@ public class ScriptHeuristic extends SignificanceHeuristic { @Override public void initialize(InternalAggregation.ReduceContext context) { - initialize(context.scriptService()); + initialize(context.scriptService().executable(script, ScriptContext.Standard.AGGS, Collections.emptyMap())); } @Override public void initialize(SearchContext context) { - initialize(context.scriptService()); + initialize(context.getQueryShardContext().getExecutableScript(script, ScriptContext.Standard.AGGS, Collections.emptyMap())); } - public void initialize(ScriptService scriptService) { - searchScript = scriptService.executable(script, ScriptContext.Standard.AGGS, Collections.emptyMap()); - searchScript.setNextVar("_subset_freq", subsetDfHolder); - searchScript.setNextVar("_subset_size", subsetSizeHolder); - searchScript.setNextVar("_superset_freq", supersetDfHolder); - searchScript.setNextVar("_superset_size", supersetSizeHolder); + public void initialize(ExecutableScript executableScript) { + this.executableScript = executableScript; + this.executableScript.setNextVar("_subset_freq", subsetDfHolder); + this.executableScript.setNextVar("_subset_size", subsetSizeHolder); + this.executableScript.setNextVar("_superset_freq", supersetDfHolder); + this.executableScript.setNextVar("_superset_size", supersetSizeHolder); } /** @@ -100,7 +99,7 @@ public class ScriptHeuristic extends SignificanceHeuristic { */ @Override public double getScore(long subsetFreq, long subsetSize, long supersetFreq, long supersetSize) { - if (searchScript == null) { + if (executableScript == null) { //In tests, wehn calling assertSearchResponse(..) the response is streamed one additional time with an arbitrary version, see assertVersionSerializable(..). // Now, for version before 1.5.0 the score is computed after streaming the response but for scripts the script does not exists yet. // assertSearchResponse() might therefore fail although there is no problem. @@ -112,7 +111,7 @@ public class ScriptHeuristic extends SignificanceHeuristic { supersetSizeHolder.value = supersetSize; subsetDfHolder.value = subsetFreq; supersetDfHolder.value = supersetFreq; - return ((Number) searchScript.run()).doubleValue(); + return ((Number) executableScript.run()).doubleValue(); } @Override @@ -171,26 +170,6 @@ public class ScriptHeuristic extends SignificanceHeuristic { return new ScriptHeuristic(script); } - public static class ScriptHeuristicBuilder implements SignificanceHeuristicBuilder { - - private Script script = null; - - public ScriptHeuristicBuilder setScript(Script script) { - this.script = script; - return this; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params builderParams) throws IOException { - builder.startObject(NAME); - builder.field(ScriptField.SCRIPT.getPreferredName()); - script.toXContent(builder, builderParams); - builder.endObject(); - return builder; - } - - } - public final class LongAccessor extends Number { public long value; @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java index 244881a5155..f044e94f05b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java @@ -26,18 +26,23 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.InternalAggregation.Type; import org.elasticsearch.search.aggregations.support.AggregationContext; - import java.io.IOException; +import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Function; public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder { @@ -182,10 +187,29 @@ public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder @Override protected ScriptedMetricAggregatorFactory doBuild(AggregationContext context, AggregatorFactory parent, Builder subfactoriesBuilder) throws IOException { - return new ScriptedMetricAggregatorFactory(name, type, initScript, mapScript, combineScript, reduceScript, params, context, - parent, subfactoriesBuilder, metaData); + + QueryShardContext queryShardContext = context.searchContext().getQueryShardContext(); + Function, ExecutableScript> executableInitScript; + if (initScript != null) { + executableInitScript = queryShardContext.getLazyExecutableScript(initScript, ScriptContext.Standard.AGGS, + Collections.emptyMap()); + } else { + executableInitScript = (p) -> null;; + } + Function, SearchScript> searchMapScript = queryShardContext.getLazySearchScript(mapScript, + ScriptContext.Standard.AGGS, Collections.emptyMap()); + Function, ExecutableScript> executableCombineScript; + if (combineScript != null) { + executableCombineScript = queryShardContext.getLazyExecutableScript(combineScript, ScriptContext.Standard.AGGS, + Collections.emptyMap()); + } else { + executableCombineScript = (p) -> null; + } + return new ScriptedMetricAggregatorFactory(name, type, searchMapScript, executableInitScript, executableCombineScript, reduceScript, + params, context, parent, subfactoriesBuilder, metaData); } + @Override protected XContentBuilder internalXContent(XContentBuilder builder, Params builderParams) throws IOException { builder.startObject(); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java index e2d3034fa11..57fd49779fc 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.metrics.scripted; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.script.Script; @@ -46,21 +47,14 @@ public class ScriptedMetricAggregator extends MetricsAggregator { private final Script reduceScript; private Map params; - protected ScriptedMetricAggregator(String name, Script initScript, Script mapScript, Script combineScript, Script reduceScript, + protected ScriptedMetricAggregator(String name, SearchScript mapScript, ExecutableScript combineScript, + Script reduceScript, Map params, AggregationContext context, Aggregator parent, List pipelineAggregators, Map metaData) throws IOException { super(name, context, parent, pipelineAggregators, metaData); this.params = params; - ScriptService scriptService = context.searchContext().scriptService(); - if (initScript != null) { - scriptService.executable(initScript, ScriptContext.Standard.AGGS, Collections.emptyMap()).run(); - } - this.mapScript = scriptService.search(context.searchContext().lookup(), mapScript, ScriptContext.Standard.AGGS, Collections.emptyMap()); - if (combineScript != null) { - this.combineScript = scriptService.executable(combineScript, ScriptContext.Standard.AGGS, Collections.emptyMap()); - } else { - this.combineScript = null; - } + this.mapScript = mapScript; + this.combineScript = combineScript; this.reduceScript = reduceScript; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java index f89e99f44b3..d6150fa8743 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java @@ -19,7 +19,9 @@ package org.elasticsearch.search.aggregations.metrics.scripted; +import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; +import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -34,22 +36,23 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; +import java.util.function.Function; public class ScriptedMetricAggregatorFactory extends AggregatorFactory { - private final Script initScript; - private final Script mapScript; - private final Script combineScript; + private final Function, SearchScript> mapScript; + private final Function, ExecutableScript> combineScript; private final Script reduceScript; private final Map params; + private final Function, ExecutableScript> initScript; - public ScriptedMetricAggregatorFactory(String name, Type type, Script initScript, Script mapScript, Script combineScript, + public ScriptedMetricAggregatorFactory(String name, Type type, Function, SearchScript> mapScript, + Function, ExecutableScript> initScript, Function, ExecutableScript> combineScript, Script reduceScript, Map params, AggregationContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, Map metaData) throws IOException { super(name, type, context, parent, subFactories, metaData); - this.initScript = initScript; this.mapScript = mapScript; + this.initScript = initScript; this.combineScript = combineScript; this.reduceScript = reduceScript; this.params = params; @@ -68,16 +71,18 @@ public class ScriptedMetricAggregatorFactory extends AggregatorFactory(); params.put("_agg", new HashMap()); } - return new ScriptedMetricAggregator(name, insertParams(initScript, params), insertParams(mapScript, params), - insertParams(combineScript, params), deepCopyScript(reduceScript, context.searchContext()), params, context, parent, - pipelineAggregators, metaData); - } - private static Script insertParams(Script script, Map params) { - if (script == null) { - return null; + final ExecutableScript initScript = this.initScript.apply(params); + final SearchScript mapScript = this.mapScript.apply(params); + final ExecutableScript combineScript = this.combineScript.apply(params); + + final Script reduceScript = deepCopyScript(this.reduceScript, context.searchContext()); + if (initScript != null) { + initScript.run(); } - return new Script(script.getScript(), script.getType(), script.getLang(), params); + return new ScriptedMetricAggregator(name, mapScript, + combineScript, reduceScript, params, context, parent, + pipelineAggregators, metaData); } private static Script deepCopyScript(Script script, SearchContext context) { @@ -98,26 +103,27 @@ public class ScriptedMetricAggregatorFactory extends AggregatorFactory originalMap = (Map) original; Map clonedMap = new HashMap<>(); - for (Entry e : originalMap.entrySet()) { + for (Map.Entry e : originalMap.entrySet()) { clonedMap.put(deepCopyParams(e.getKey(), context), deepCopyParams(e.getValue(), context)); } clone = (T) clonedMap; } else if (original instanceof List) { List originalList = (List) original; - List clonedList = new ArrayList(); + List clonedList = new ArrayList<>(); for (Object o : originalList) { clonedList.add(deepCopyParams(o, context)); } clone = (T) clonedList; } else if (original instanceof String || original instanceof Integer || original instanceof Long || original instanceof Short - || original instanceof Byte || original instanceof Float || original instanceof Double || original instanceof Character - || original instanceof Boolean) { + || original instanceof Byte || original instanceof Float || original instanceof Double || original instanceof Character + || original instanceof Boolean) { clone = original; } else { throw new SearchParseException(context, - "Can only clone primitives, String, ArrayList, and HashMap. Found: " + original.getClass().getCanonicalName(), null); + "Can only clone primitives, String, ArrayList, and HashMap. Found: " + original.getClass().getCanonicalName(), null); } return clone; } + } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java index d8f3adc8044..9f740b695b7 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java @@ -530,20 +530,17 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder parent, Builder subfactoriesBuilder) throws IOException { - List scriptFields = null; - if (this.scriptFields != null) { - scriptFields = new ArrayList<>(); - for (ScriptField field : this.scriptFields) { - SearchScript searchScript = context.searchContext().scriptService().search( - context.searchContext().lookup(), field.script(), ScriptContext.Standard.SEARCH, Collections.emptyMap()); - scriptFields.add(new ScriptFieldsContext.ScriptField( - field.fieldName(), searchScript, field.ignoreFailure())); + List fields = new ArrayList<>(); + if (scriptFields != null) { + for (ScriptField field : scriptFields) { + SearchScript searchScript = context.searchContext().getQueryShardContext().getSearchScript(field.script(), + ScriptContext.Standard.SEARCH, Collections.emptyMap()); + fields.add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField( + field.fieldName(), searchScript, field.ignoreFailure())); } - } else { - scriptFields = Collections.emptyList(); } return new TopHitsAggregatorFactory(name, type, from, size, explain, version, trackScores, sorts, highlightBuilder, - storedFieldsContext, fieldDataFields, scriptFields, fetchSourceContext, context, + storedFieldsContext, fieldDataFields, fields, fetchSourceContext, context, parent, subfactoriesBuilder, metaData); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java index 41fe2fb0a64..783c3baa7a4 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java @@ -56,8 +56,8 @@ public class TopHitsAggregatorFactory extends AggregatorFactory> sorts, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext, List docValueFields, List scriptFields, FetchSourceContext fetchSourceContext, - AggregationContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, - Map metaData) throws IOException { + AggregationContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, Map metaData) + throws IOException { super(name, type, context, parent, subFactories, metaData); this.from = from; this.size = size; @@ -96,7 +96,7 @@ public class TopHitsAggregatorFactory extends AggregatorFactory { @Override public SortFieldAndFormat build(QueryShardContext context) throws IOException { - final SearchScript searchScript = context.getScriptService().search( - context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap()); + final SearchScript searchScript = context.getSearchScript(script, ScriptContext.Standard.SEARCH, Collections.emptyMap()); MultiValueMode valueMode = null; if (sortMode != null) { diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java index a9f5accb918..41168852d2c 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java @@ -55,6 +55,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; public final class PhraseSuggester extends Suggester { private final BytesRef SEPARATOR = new BytesRef(" "); @@ -109,7 +110,7 @@ public final class PhraseSuggester extends Suggester { response.addTerm(resultEntry); final BytesRefBuilder byteSpare = new BytesRefBuilder(); - final CompiledScript collateScript = suggestion.getCollateQueryScript(); + final Function, ExecutableScript> collateScript = suggestion.getCollateQueryScript(); final boolean collatePrune = (collateScript != null) && suggestion.collatePrune(); for (int i = 0; i < checkerResult.corrections.length; i++) { Correction correction = checkerResult.corrections[i]; @@ -121,8 +122,7 @@ public final class PhraseSuggester extends Suggester { final Map vars = suggestion.getCollateScriptParams(); vars.put(SUGGESTION_TEMPLATE_VAR_NAME, spare.toString()); QueryShardContext shardContext = suggestion.getShardContext(); - ScriptService scriptService = shardContext.getScriptService(); - final ExecutableScript executable = scriptService.executable(collateScript, vars); + final ExecutableScript executable = collateScript.apply(vars); final BytesReference querySource = (BytesReference) executable.run(); try (XContentParser parser = XContentFactory.xContent(querySource).createParser(querySource)) { Optional innerQueryBuilder = shardContext.newParseContext(parser).parseInnerQueryBuilder(); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java index b0cd6a20499..7ec7fdda069 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; @@ -56,6 +57,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.Set; +import java.util.function.Function; /** * Defines the actual suggest command for phrase suggestions ( phrase). @@ -633,8 +635,8 @@ public class PhraseSuggestionBuilder extends SuggestionBuilder, ExecutableScript> compiledScript = context.getLazyExecutableScript(this.collateQuery, + ScriptContext.Standard.SEARCH, Collections.emptyMap()); suggestionContext.setCollateQueryScript(compiledScript); if (this.collateParams != null) { suggestionContext.setCollateScriptParams(this.collateParams); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java index e2d7ff1c41d..960dca419f7 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; @@ -31,6 +32,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; class PhraseSuggestionContext extends SuggestionContext { static final boolean DEFAULT_COLLATE_PRUNE = false; @@ -52,7 +54,7 @@ class PhraseSuggestionContext extends SuggestionContext { private boolean requireUnigram = DEFAULT_REQUIRE_UNIGRAM; private BytesRef preTag; private BytesRef postTag; - private CompiledScript collateQueryScript; + private Function, ExecutableScript> collateQueryScript; private boolean prune = DEFAULT_COLLATE_PRUNE; private List generators = new ArrayList<>(); private Map collateScriptParams = new HashMap<>(1); @@ -192,11 +194,11 @@ class PhraseSuggestionContext extends SuggestionContext { return postTag; } - CompiledScript getCollateQueryScript() { + Function, ExecutableScript> getCollateQueryScript() { return collateQueryScript; } - void setCollateQueryScript(CompiledScript collateQueryScript) { + void setCollateQueryScript( Function, ExecutableScript> collateQueryScript) { this.collateQueryScript = collateQueryScript; } diff --git a/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java index 062f8c01f42..2a961d58928 100644 --- a/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java +++ b/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java @@ -46,8 +46,7 @@ public class SearchSlowLogTests extends ESSingleNodeTestCase { protected SearchContext createSearchContext(IndexService indexService) { BigArrays bigArrays = indexService.getBigArrays(); ThreadPool threadPool = indexService.getThreadPool(); - ScriptService scriptService = node().injector().getInstance(ScriptService.class); - return new TestSearchContext(threadPool, bigArrays, scriptService, indexService) { + return new TestSearchContext(threadPool, bigArrays, indexService) { @Override public ShardSearchRequest request() { return new ShardSearchRequest() { diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 3b3bdf31b75..d913fc37aae 100644 --- a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -46,9 +46,10 @@ public class QueryShardContextTests extends ESTestCase { IndexSettings indexSettings = new IndexSettings(indexMetadata.build(), Settings.EMPTY); MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexSettings()).thenReturn(indexSettings); + final long nowInMillis = randomPositiveLong(); QueryShardContext context = new QueryShardContext( - indexSettings, null, null, mapperService, null, null, null, null, null, null - ); + indexSettings, null, null, mapperService, null, null, null, null, null, null, + () -> nowInMillis); context.setAllowUnmappedFields(false); MappedFieldType fieldType = new TextFieldMapper.TextFieldType(); diff --git a/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java index 63b8101515e..b86babb5a31 100644 --- a/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java @@ -51,7 +51,7 @@ public class ScriptQueryBuilderTests extends AbstractQueryTestCase params = new HashMap<>(); + params.put("fieldname", "d"); + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(dateHistogram("histo").field("d") + .script(new Script(DateScriptMocks.PlusOneMonthScript.NAME, ScriptType.INLINE, "native", params)) + .dateHistogramInterval(DateHistogramInterval.MONTH)).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(dateHistogram("histo").field("d").dateHistogramInterval(DateHistogramInterval.MONTH)).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java index 1e250681004..a6e0273beff 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; @@ -866,4 +867,51 @@ public class DateRangeIT extends ESIntegTestCase { assertThat(buckets.get(0).getDocCount(), equalTo(0L)); assertThat(buckets.get(0).getAggregations().asList().isEmpty(), is(true)); } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "date", "type=date") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, + client().prepareIndex("cache_test_idx", "type", "1") + .setSource(jsonBuilder().startObject().field("date", date(1, 1)).endObject()), + client().prepareIndex("cache_test_idx", "type", "2") + .setSource(jsonBuilder().startObject().field("date", date(2, 1)).endObject())); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + Map params = new HashMap<>(); + params.put("fieldname", "date"); + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(dateRange("foo").field("date") + .script(new Script(DateScriptMocks.PlusOneMonthScript.NAME, ScriptType.INLINE, "native", params)) + .addRange(new DateTime(2012, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC), new DateTime(2013, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC))) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(dateRange("foo").field("date") + .addRange(new DateTime(2012, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC), new DateTime(2013, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC))) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java index ed43ba8bd63..2393512c1a3 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java @@ -21,13 +21,16 @@ package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScoreAccessor; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; +import org.elasticsearch.search.aggregations.bucket.LongTermsIT.CustomScriptPlugin; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -56,7 +59,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.scriptFunction; -import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; @@ -1179,4 +1181,42 @@ public class DoubleTermsIT extends AbstractTermsTestCase { public void testOtherDocCount() { testOtherDocCount(SINGLE_VALUED_FIELD_NAME, MULTI_VALUED_FIELD_NAME); } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=float") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1.5), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2.5)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation( + terms("terms").field("d").script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null))).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(terms("terms").field("d")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java index e24f0d39d4b..0c4de629c6e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.bucket; import com.carrotsearch.hppc.LongHashSet; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptPlugin; @@ -995,4 +996,43 @@ public class HistogramIT extends ESIntegTestCase { assertEquals(0.05, (double) buckets.get(1).getKey(), 0.01d); assertEquals(1, buckets.get(1).getDocCount()); } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=float") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("d", -0.6), + client().prepareIndex("cache_test_idx", "type", "2").setSource("d", 0.1)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(histogram("histo").field("d") + .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())).interval(0.7).offset(0.05)).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(histogram("histo").field("d").interval(0.7).offset(0.05)) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java index 6d5e190b542..635b1635cc2 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java @@ -21,12 +21,15 @@ package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; +import org.elasticsearch.search.aggregations.bucket.StringTermsIT.CustomScriptPlugin; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -53,7 +56,6 @@ import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats; import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; @@ -1136,4 +1138,42 @@ public class LongTermsIT extends AbstractTermsTestCase { public void testOtherDocCount() { testOtherDocCount(SINGLE_VALUED_FIELD_NAME, MULTI_VALUED_FIELD_NAME); } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation( + terms("terms").field("d").script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null))).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(terms("terms").field("d")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java index d3e934d875f..c64fc213096 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeIT.java @@ -20,9 +20,11 @@ package org.elasticsearch.search.aggregations.bucket; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; @@ -32,21 +34,25 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.Matchers; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.script.ScriptService.ScriptType; +import static org.elasticsearch.search.aggregations.AggregationBuilders.dateRange; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.range; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -907,4 +913,46 @@ public class RangeIT extends ESIntegTestCase { assertThat(buckets.get(0).getDocCount(), equalTo(0L)); } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "i", "type=integer") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, + client().prepareIndex("cache_test_idx", "type", "1").setSource(jsonBuilder().startObject().field("i", 1).endObject()), + client().prepareIndex("cache_test_idx", "type", "2").setSource(jsonBuilder().startObject().field("i", 2).endObject())); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + Map params = new HashMap<>(); + params.put("fieldname", "date"); + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation( + range("foo").field("i").script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null)).addRange(0, 10)) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(range("foo").field("i").addRange(0, 10)).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java index fab1f8b7d3e..150f4a74a96 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; @@ -547,4 +548,43 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase { SharedSignificantTermsTestMethods.aggregateAndCheckFromSeveralShards(this); } + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + ScriptHeuristic scriptHeuristic = getScriptSignificanceHeuristic(); + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(significantTerms("foo").field("s").significanceHeuristic(scriptHeuristic)).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(significantTerms("foo").field("s")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } + } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java index 8ccccf3f33a..df1741dc8b0 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java @@ -24,11 +24,13 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; @@ -61,7 +63,6 @@ import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; import static org.elasticsearch.search.aggregations.AggregationBuilders.count; import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats; @@ -1511,4 +1512,44 @@ public class StringTermsIT extends AbstractTermsTestCase { public void testOtherDocCount() { testOtherDocCount(SINGLE_VALUED_FIELD_NAME, MULTI_VALUED_FIELD_NAME); } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=keyword") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", "foo"), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", "bar")); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation( + terms("terms").field("d").script(new Script("'foo_' + _value", ScriptType.INLINE, CustomScriptPlugin.NAME, null))) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(terms("terms").field("d")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java index 6b7e51cc90f..f65d7eb1f7b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; @@ -93,7 +94,9 @@ public class ExtendedBoundsTests extends ESTestCase { public void testParseAndValidate() { long now = randomLong(); SearchContext context = mock(SearchContext.class); - when(context.nowInMillis()).thenReturn(now); + QueryShardContext qsc = mock(QueryShardContext.class); + when(context.getQueryShardContext()).thenReturn(qsc); + when(qsc.nowInMillis()).thenReturn(now); FormatDateTimeFormatter formatter = Joda.forPattern("dateOptionalTime"); DocValueFormat format = new DocValueFormat.DateTime(formatter, DateTimeZone.UTC); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java index ccf03df37fc..37a3d126e3f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java @@ -56,7 +56,9 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.filter; import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -353,6 +355,44 @@ public class AvgIT extends AbstractNumericTestCase { } } + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(avg("foo").field("d").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null))).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(avg("foo").field("d")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } + /** * Mock plugin for the {@link ExtractFieldScriptEngine} */ diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java index cff1fa746dc..35e46b50785 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -41,10 +42,10 @@ import java.util.function.Function; import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.script.ScriptService.ScriptType; import static org.elasticsearch.search.aggregations.AggregationBuilders.cardinality; import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -445,4 +446,44 @@ public class CardinalityIT extends ESIntegTestCase { assertCount(count, 2); } } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation( + cardinality("foo").field("d").script(new Script("_value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap()))) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(cardinality("foo").field("d")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java index 6f4f891ea65..319a7d64230 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; @@ -46,7 +47,9 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.missing; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -630,4 +633,44 @@ public class ExtendedStatsIT extends AbstractNumericTestCase { assertThat(stats.getStdDeviationBound(ExtendedStats.Bounds.LOWER), equalTo(stats.getAvg() - (stats.getStdDeviation() * sigma))); } + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(extendedStats("foo").field("d") + .script(new Script("_value + 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null))) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(extendedStats("foo").field("d")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } + } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java index 57bdc7d5dfc..46130fca21f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; @@ -49,7 +50,9 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.percentileRanks; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -530,4 +533,45 @@ public class HDRPercentileRanksIT extends AbstractNumericTestCase { } } + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client() + .prepareSearch("cache_test_idx").setSize(0).addAggregation(percentileRanks("foo").method(PercentilesMethod.HDR).field("d") + .values(50.0).script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(percentileRanks("foo").method(PercentilesMethod.HDR).field("d").values(50.0)).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } + } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java index 8112551f53c..784b4816393 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; @@ -49,7 +50,9 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -521,4 +524,45 @@ public class HDRPercentilesIT extends AbstractNumericTestCase { } } + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(percentiles("foo").method(PercentilesMethod.HDR).field("d").percentiles(50.0) + .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(percentiles("foo").method(PercentilesMethod.HDR).field("d").percentiles(50.0)).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } + } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java index d4ac8835ee0..c3e693b5307 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; @@ -44,7 +45,9 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.max; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -352,4 +355,43 @@ public class MaxIT extends AbstractNumericTestCase { } } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation( + max("foo").field("d").script(new Script("_value + 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(max("foo").field("d")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java index 56c12fbc77f..93402534619 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; @@ -44,7 +45,9 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.min; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -365,4 +368,43 @@ public class MinIT extends AbstractNumericTestCase { } } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation( + min("foo").field("d").script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(min("foo").field("d")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java index e1800b2f9f1..c1a5aad556f 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java @@ -933,4 +933,33 @@ public class ScriptedMetricIT extends ESIntegTestCase { assertThat(aggregationResult.size(), equalTo(1)); assertThat(aggregationResult.get(0), equalTo(0)); } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + Script mapScript = new Script("_agg['count'] = 1", ScriptType.INLINE, CustomScriptPlugin.NAME, null); + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(scriptedMetric("foo").mapScript(mapScript)).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java index 510df4c572b..7ab44c49447 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java @@ -22,6 +22,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; @@ -47,7 +48,9 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -485,4 +488,42 @@ public class StatsIT extends AbstractNumericTestCase { } assertThat("Not all shards are initialized", response.getSuccessfulShards(), equalTo(response.getTotalShards())); } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation( + stats("foo").field("d").script(new Script("_value + 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, null))).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(stats("foo").field("d")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java index 186bc8fb27b..3ff00529655 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java @@ -55,7 +55,9 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -347,6 +349,44 @@ public class SumIT extends AbstractNumericTestCase { } } + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(sum("foo").field("d").script(new Script("", ScriptType.INLINE, FieldValueScriptEngine.NAME, null))).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(sum("foo").field("d")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } + /** * Mock plugin for the {@link ExtractFieldScriptEngine} */ diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java index e50b89d8b96..3a9ab247f5a 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java @@ -20,10 +20,9 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin; @@ -43,8 +42,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; - import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; @@ -53,7 +50,9 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.percentileRanks; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -484,4 +483,42 @@ public class TDigestPercentileRanksIT extends AbstractNumericTestCase { } } + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(percentileRanks("foo").field("d").values(50.0) + .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(percentileRanks("foo").field("d").values(50.0)).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } + } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java index b0268351954..5f3fbb7e869 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.metrics; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; @@ -50,7 +51,9 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.percentiles; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -465,4 +468,43 @@ public class TDigestPercentilesIT extends AbstractNumericTestCase { } } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(percentiles("foo").field("d") + .percentiles(50.0).script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(percentiles("foo").field("d").percentiles(50.0)).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index 45d44c863a4..383d25c4011 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.query.QueryBuilders; @@ -33,6 +34,7 @@ import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.SearchHits; @@ -993,4 +995,42 @@ public class TopHitsIT extends ESIntegTestCase { } } } + + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(topHits("foo").scriptField("bar", new Script("5", ScriptType.INLINE, CustomScriptPlugin.NAME, null))).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(topHits("foo")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java index 7de37fdb6a8..90e56249ff3 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java @@ -28,9 +28,7 @@ import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptEngineRegistry; import org.elasticsearch.script.ScriptEngineService; -import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.aggregations.bucket.global.Global; @@ -43,14 +41,15 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.count; import static org.elasticsearch.search.aggregations.AggregationBuilders.global; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -209,6 +208,45 @@ public class ValueCountIT extends ESIntegTestCase { assertThat(valueCount.getValue(), equalTo(20L)); } + /** + * Make sure that a request using a script does not get cached and a request + * not using a script does get cached. + */ + public void testDontCacheScripts() throws Exception { + assertAcked(prepareCreate("cache_test_idx").addMapping("type", "d", "type=long") + .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) + .get()); + indexRandom(true, client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1), + client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2)); + + // Make sure we are starting with a clear cache + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // Test that a request using a script does not get cached + SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0) + .addAggregation(count("foo").field("d").script(new Script("value", ScriptType.INLINE, FieldValueScriptEngine.NAME, null))) + .get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(0L)); + + // To make sure that the cache is working test that a request not using + // a script is cached + r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(count("foo").field("d")).get(); + assertSearchResponse(r); + + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getHitCount(), equalTo(0L)); + assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache() + .getMissCount(), equalTo(1L)); + } + /** * Mock plugin for the {@link FieldValueScriptEngine} */ diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index ed272040f51..78e05d4207b 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -49,9 +49,6 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.fetch.subphase.highlight.AbstractHighlighterBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.Field; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.Order; import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight.FieldOptions; @@ -294,7 +291,7 @@ public class HighlightBuilderTests extends ESTestCase { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry, - null, null, null) { + null, null, null, System::currentTimeMillis) { @Override public MappedFieldType fieldMapper(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java index 1af4a2b1788..24d97f3b5ac 100644 --- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java @@ -156,12 +156,13 @@ public class QueryRescoreBuilderTests extends ESTestCase { * than the test builder */ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, IOException { + final long nowInMillis = randomPositiveLong(); Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry, - null, null, null) { + null, null, null, () -> nowInMillis) { @Override public MappedFieldType fieldMapper(String name) { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name); diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 84f83c46e1a..6293401ebdd 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -24,7 +24,6 @@ import org.apache.lucene.util.Accountable; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -235,8 +234,9 @@ public abstract class AbstractSortTestCase> extends EST public void onCache(ShardId shardId, Accountable accountable) { } }); + long nowInMillis = randomPositiveLong(); return new QueryShardContext(idxSettings, bitsetFilterCache, ifds, null, null, scriptService, - indicesQueriesRegistry, null, null, null) { + indicesQueriesRegistry, null, null, null, () -> nowInMillis) { @Override public MappedFieldType fieldMapper(String name) { return provideMappedFieldType(name); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java index 4a19bcb4b93..f81d5c7e053 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java @@ -120,9 +120,7 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder> transportClientPlugins() { return Collections.singleton(PercolatorPlugin.class); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index b21c131a625..828f800f34e 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.percolator; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -31,12 +32,20 @@ import org.elasticsearch.index.query.MultiMatchQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESSingleNodeTestCase; +import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; @@ -49,7 +58,13 @@ import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery; import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery; import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.percolator.PercolateSourceBuilder.docBuilder; +import static org.elasticsearch.percolator.PercolatorTestUtil.assertMatchCount; +import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray; +import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -58,7 +73,33 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return Collections.singleton(PercolatorPlugin.class); + return Arrays.asList(PercolatorPlugin.class, CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + @Override + protected Map, Object>> pluginScripts() { + Map, Object>> scripts = new HashMap<>(); + scripts.put("1==1", vars -> Boolean.TRUE); + return scripts; + } + } + + public void testPercolateScriptQuery() throws IOException { + client().admin().indices().prepareCreate("index").addMapping("type", "query", "type=percolator").get(); + ensureGreen(); + client().prepareIndex("index", "type", "1") + .setSource(jsonBuilder().startObject().field("query", QueryBuilders.scriptQuery( + new Script("1==1", ScriptService.ScriptType.INLINE, CustomScriptPlugin.NAME, null))).endObject()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .execute().actionGet(); + PercolateResponse response = preparePercolate(client()) + .setIndices("index").setDocumentType("type") + .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "b").endObject())) + .execute().actionGet(); + assertMatchCount(response, 1L); + assertThat(response.getMatches(), arrayWithSize(1)); + assertThat(convertFromTextArray(response.getMatches(), "index"), arrayContainingInAnyOrder("1")); } public void testPercolatorQuery() throws Exception { diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index 6225f5fa5d0..711c74375f1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -580,11 +580,23 @@ public abstract class AbstractQueryTestCase> public void testToQuery() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTQUERIES; runs++) { QueryShardContext context = createShardContext(); + assert context.isCachable(); context.setAllowUnmappedFields(true); QB firstQuery = createTestQueryBuilder(); QB controlQuery = copyQuery(firstQuery); setSearchContext(randomTypes, context); // only set search context for toQuery to be more realistic - Query firstLuceneQuery = rewriteQuery(firstQuery, context).toQuery(context); + /* we use a private rewrite context here since we want the most realistic way of asserting that we are cachabel or not. + * We do it this way in SearchService where + * we first rewrite the query with a private context, then reset the context and then build the actual lucene query*/ + QueryBuilder rewritten = rewriteQuery(firstQuery, new QueryShardContext(context)); + Query firstLuceneQuery = rewritten.toQuery(context); + if (isCachable(firstQuery)) { + assertTrue("query was marked as not cacheable in the context but this test indicates it should be cacheable: " + + firstQuery.toString(), context.isCachable()); + } else { + assertFalse("query was marked as cacheable in the context but this test indicates it should not be cacheable: " + + firstQuery.toString(), context.isCachable()); + } assertNotNull("toQuery should not return null", firstLuceneQuery); assertLuceneQuery(firstQuery, firstLuceneQuery, context); //remove after assertLuceneQuery since the assertLuceneQuery impl might access the context as well @@ -636,6 +648,10 @@ public abstract class AbstractQueryTestCase> return rewritten; } + protected boolean isCachable(QB queryBuilder) { + return true; + } + /** * Few queries allow you to set the boost and queryName on the java api, although the corresponding parser * doesn't parse them as they are not supported. This method allows to disable boost and queryName related tests for those queries. @@ -1020,6 +1036,7 @@ public abstract class AbstractQueryTestCase> private final BitsetFilterCache bitsetFilterCache; private final ScriptService scriptService; private final Client client; + private final long nowInMillis = randomPositiveLong(); ServiceHolder(Settings nodeSettings, Settings indexSettings, Collection> plugins, AbstractQueryTestCase testCase) throws IOException { @@ -1098,7 +1115,7 @@ public abstract class AbstractQueryTestCase> QueryShardContext createShardContext() { ClusterState state = ClusterState.builder(new ClusterName("_name")).build(); return new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, - scriptService, indicesQueriesRegistry, this.client, null, state); + scriptService, indicesQueriesRegistry, this.client, null, state, () -> nowInMillis); } ScriptModule createScriptModule(List scriptPlugins) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 4916d7df2f6..e440af4e9b9 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -283,7 +283,7 @@ public abstract class ESSingleNodeTestCase extends ESTestCase { BigArrays bigArrays = indexService.getBigArrays(); ThreadPool threadPool = indexService.getThreadPool(); ScriptService scriptService = node().injector().getInstance(ScriptService.class); - return new TestSearchContext(threadPool, bigArrays, scriptService, indexService); + return new TestSearchContext(threadPool, bigArrays, indexService); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index c6a1f64820b..c8feb8242be 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -37,7 +37,6 @@ import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchExtBuilder; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.aggregations.SearchContextAggregations; @@ -77,7 +76,6 @@ public class TestSearchContext extends SearchContext { final Counter timeEstimateCounter = Counter.newCounter(); final QuerySearchResult queryResult = new QuerySearchResult(); final QueryShardContext queryShardContext; - ScriptService scriptService; ParsedQuery originalQuery; ParsedQuery postFilter; Query query; @@ -91,7 +89,7 @@ public class TestSearchContext extends SearchContext { private final long originNanoTime = System.nanoTime(); private final Map searchExtBuilders = new HashMap<>(); - public TestSearchContext(ThreadPool threadPool, BigArrays bigArrays, ScriptService scriptService, IndexService indexService) { + public TestSearchContext(ThreadPool threadPool, BigArrays bigArrays, IndexService indexService) { super(ParseFieldMatcher.STRICT); this.bigArrays = bigArrays.withCircuitBreaking(); this.indexService = indexService; @@ -99,7 +97,6 @@ public class TestSearchContext extends SearchContext { this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache(); this.threadPool = threadPool; this.indexShard = indexService.getShardOrNull(0); - this.scriptService = scriptService; queryShardContext = indexService.newQueryShardContext(); } @@ -111,7 +108,6 @@ public class TestSearchContext extends SearchContext { this.threadPool = null; this.fixedBitSetFilterCache = null; this.indexShard = null; - scriptService = null; this.queryShardContext = queryShardContext; } @@ -169,11 +165,6 @@ public class TestSearchContext extends SearchContext { return originNanoTime; } - @Override - protected long nowInMillisImpl() { - return 0; - } - @Override public ScrollContext scrollContext() { return null; @@ -299,11 +290,6 @@ public class TestSearchContext extends SearchContext { return null; } - @Override - public ScriptService scriptService() { - return scriptService; - } - @Override public BigArrays bigArrays() { return bigArrays; diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java index d6cd3eea5ac..8353a58e969 100644 --- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java +++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java @@ -33,8 +33,9 @@ import org.elasticsearch.test.TestSearchContext; public class MockSearchServiceTests extends ESTestCase { public void testAssertNoInFlightContext() { + final long nowInMillis = randomPositiveLong(); SearchContext s = new TestSearchContext(new QueryShardContext(new IndexSettings(IndexMetaData.PROTO, Settings.EMPTY), null, null, - null, null, null, null, null, null, null)) { + null, null, null, null, null, null, null, () -> nowInMillis)) { @Override public SearchShardTarget shardTarget() { return new SearchShardTarget("node", new Index("idx", "ignored"), 0);