honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Scripting: Deprecate general cache settings (#55753) (#58283) 

Backport: ef543b0
This commit is contained in:
Stuart Tettemer 2020-06-18 11:54:23 -06:00 committed by GitHub
parent 1c1a6d4ec8
commit 20abba8433
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 629 additions and 390 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View File

@ -222,7 +222,7 @@ class ClusterFormationTasks {
if (distro.equals("oss")) { if (distro.equals("oss")) {
snapshotProject = "oss-" + snapshotProject snapshotProject = "oss-" + snapshotProject
} }
BwcVersions.UnreleasedVersionInfo unreleasedInfo = null BwcVersions.UnreleasedVersionInfo unreleasedInfo = null
if (project.hasProperty('bwcVersions')) { if (project.hasProperty('bwcVersions')) {
@ -420,7 +420,11 @@ class ClusterFormationTasks {
esConfig['cluster.routing.allocation.disk.watermark.flood_stage'] = '1b' esConfig['cluster.routing.allocation.disk.watermark.flood_stage'] = '1b'
} }
// increase script compilation limit since tests can rapid-fire script compilations // increase script compilation limit since tests can rapid-fire script compilations
esConfig['script.max_compilations_rate'] = '2048/1m' if (node.nodeVersion.onOrAfter('7.9.0')) {
esConfig['script.disable_max_compilations_rate'] = 'true'
} else {
esConfig['script.max_compilations_rate'] = '2048/1m'
}
// Temporarily disable the real memory usage circuit breaker. It depends on real memory usage which we have no full control // Temporarily disable the real memory usage circuit breaker. It depends on real memory usage which we have no full control
// over and the REST client will not retry on circuit breaking exceptions yet (see #31986 for details). Once the REST client // over and the REST client will not retry on circuit breaking exceptions yet (see #31986 for details). Once the REST client
// can retry on circuit breaking exceptions, we can revert again to the default configuration. // can retry on circuit breaking exceptions, we can revert again to the default configuration.

6
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
View File

@ -1093,7 +1093,11 @@ public class ElasticsearchNode implements TestClusterConfiguration {
baseConfig.put("cluster.routing.allocation.disk.watermark.low", "1b"); baseConfig.put("cluster.routing.allocation.disk.watermark.low", "1b");
baseConfig.put("cluster.routing.allocation.disk.watermark.high", "1b"); baseConfig.put("cluster.routing.allocation.disk.watermark.high", "1b");
// increase script compilation limit since tests can rapid-fire script compilations // increase script compilation limit since tests can rapid-fire script compilations
baseConfig.put("script.max_compilations_rate", "2048/1m"); if (getVersion().onOrAfter(Version.fromString("7.9.0"))) {
baseConfig.put("script.disable_max_compilations_rate", "true");
} else {
baseConfig.put("script.max_compilations_rate", "2048/1m");
}
if (getVersion().getMajor() >= 6) { if (getVersion().getMajor() >= 6) {
baseConfig.put("cluster.routing.allocation.disk.watermark.flood_stage", "1b"); baseConfig.put("cluster.routing.allocation.disk.watermark.flood_stage", "1b");
} }

4
distribution/docker/docker-compose.yml
View File

@ -15,7 +15,6 @@ services:
- cluster.routing.allocation.disk.watermark.low=1b - cluster.routing.allocation.disk.watermark.low=1b
- cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.high=1b
- cluster.routing.allocation.disk.watermark.flood_stage=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b
- script.max_compilations_rate=2048/1m
- node.store.allow_mmap=false - node.store.allow_mmap=false
- xpack.security.enabled=true - xpack.security.enabled=true
- xpack.security.transport.ssl.enabled=true - xpack.security.transport.ssl.enabled=true
@ -55,7 +54,6 @@ services:
- cluster.routing.allocation.disk.watermark.low=1b - cluster.routing.allocation.disk.watermark.low=1b
- cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.high=1b
- cluster.routing.allocation.disk.watermark.flood_stage=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b
- script.max_compilations_rate=2048/1m
- node.store.allow_mmap=false - node.store.allow_mmap=false
- xpack.security.enabled=true - xpack.security.enabled=true
- xpack.security.transport.ssl.enabled=true - xpack.security.transport.ssl.enabled=true
@ -95,7 +93,6 @@ services:
- cluster.routing.allocation.disk.watermark.low=1b - cluster.routing.allocation.disk.watermark.low=1b
- cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.high=1b
- cluster.routing.allocation.disk.watermark.flood_stage=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b
- script.max_compilations_rate=2048/1m
- node.store.allow_mmap=false - node.store.allow_mmap=false
volumes: volumes:
- ./build/oss-repo:/tmp/es-repo - ./build/oss-repo:/tmp/es-repo
@ -120,7 +117,6 @@ services:
- cluster.routing.allocation.disk.watermark.low=1b - cluster.routing.allocation.disk.watermark.low=1b
- cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.high=1b
- cluster.routing.allocation.disk.watermark.flood_stage=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b
- script.max_compilations_rate=2048/1m
- node.store.allow_mmap=false - node.store.allow_mmap=false
volumes: volumes:
- ./build/oss-repo:/tmp/es-repo - ./build/oss-repo:/tmp/es-repo

1
docs/reference/migration/index.asciidoc
View File

@ -11,6 +11,7 @@ For information about how to upgrade your cluster, see <<setup-upgrade>>.
-- --
include::migrate_7_9.asciidoc[]
include::migrate_7_8.asciidoc[] include::migrate_7_8.asciidoc[]
include::migrate_7_7.asciidoc[] include::migrate_7_7.asciidoc[]
include::migrate_7_6.asciidoc[] include::migrate_7_6.asciidoc[]

71
docs/reference/migration/migrate_7_9.asciidoc Normal file
View File

@ -0,0 +1,71 @@
[[breaking-changes-7.9]]
== Breaking changes in 7.9
++++
<titleabbrev>7.9</titleabbrev>
++++
This section discusses the changes that you need to be aware of when migrating
your application to {es} 7.9.
See also <<release-highlights>> and <<es-release-notes>>.
* <<breaking_79_script_cache_changes>>
//NOTE: The notable-breaking-changes tagged regions are re-used in the
//Installation and Upgrade Guide
//tag::notable-breaking-changes[]
[discrete]
[[breaking_79_script_cache_changes]]
=== Script cache changes
[[deprecate_general_script_cache_size]]
.The `script.cache.max_size` setting is deprecated.
[%collapsible]
====
*Details* +
The `script.cache.max_size` setting is deprecated. In {es} 8.0.0, this is
set per-context.
*Impact* +
To avoid deprecation warnings, discontinue use of the `script.cache.max_size`
setting. You may use `script.context.$CONTEXT.context_max_size` for the particular context.
For example, for the `ingest` context, use `script.context.ingest.context_max_size`.
====
[discrete]
[[deprecate_general_script_expire]]
.The `script.cache.expire` setting is deprecated.
[%collapsible]
====
*Details* +
The `script.cache.expire` setting is deprecated. In {es} 8.0.0, this is
set per-context.
*Impact* +
To avoid deprecation warnings, discontinue use of the `script.cache.expire`
setting. You may use `script.context.$CONTEXT.cache_expire` for the particular context.
For example, for the `update` context, use `script.context.update.cache_expire`.
====
[discrete]
[[deprecate_general_script_compile_rate]]
.The `script.max_compilations_rate` setting is deprecated.
[%collapsible]
====
*Details* +
The `script.max_compilations_rate` setting is deprecated. In {es} 8.0.0, this is
set per-context.
*Impact* +
To avoid deprecation warnings, discontinue use of the `script.max_compilations_rate`
setting. You may use `script.context.$CONTEXT.max_compilations_rate` for the particular
context. For example, for the `processor_conditional` context, use
`script.context.processor_conditional.max_compilations_rate`.
====
//end::notable-breaking-changes[]

6
docs/reference/modules/indices/circuit_breaker.asciidoc
View File

@ -111,8 +111,8 @@ within a period of time.
See the "prefer-parameters" section of the <<modules-scripting-using,scripting>> See the "prefer-parameters" section of the <<modules-scripting-using,scripting>>
documentation for more information. documentation for more information.
`script.max_compilations_rate`:: `script.context.$CONTEXT.max_compilations_rate`::
Limit for the number of unique dynamic scripts within a certain interval Limit for the number of unique dynamic scripts within a certain interval
that are allowed to be compiled. Defaults to 75/5m, meaning 75 every 5 that are allowed to be compiled for a given context. Defaults to `75/5m`,
minutes. meaning 75 every 5 minutes.

2
docs/reference/release-notes.asciidoc
View File

@ -6,6 +6,7 @@
This section summarizes the changes in each release. This section summarizes the changes in each release.
* <<release-notes-7.9.0>>
* <<release-notes-7.8.0>> * <<release-notes-7.8.0>>
* <<release-notes-7.7.1>> * <<release-notes-7.7.1>>
* <<release-notes-7.7.0>> * <<release-notes-7.7.0>>
@ -34,6 +35,7 @@ This section summarizes the changes in each release.
-- --
include::release-notes/7.9.asciidoc[]
include::release-notes/7.8.asciidoc[] include::release-notes/7.8.asciidoc[]
include::release-notes/7.7.asciidoc[] include::release-notes/7.7.asciidoc[]
include::release-notes/7.6.asciidoc[] include::release-notes/7.6.asciidoc[]

11
docs/reference/release-notes/7.9.asciidoc Normal file
View File

@ -0,0 +1,11 @@
[[release-notes-7.9.0]]
== {es} version 7.9.0
Also see <<breaking-changes-7.9,Breaking changes in 7.9>>.
[[breaking-7.9.0]]
[float]
=== Breaking changes
Script Cache::
* Script cache size and rate limiting are per-context {pull}55753[#55753] (issue: {issue}50152[#50152])

20
docs/reference/release-notes/highlights.asciidoc
View File

@ -3,16 +3,16 @@
coming[{minor-version}] coming[{minor-version}]
Here are the highlights of what's new and improved in {es} {minor-version}! Here are the highlights of what's new and improved in {es} {minor-version}!
ifeval::["{release-state}"!="unreleased"] ifeval::["{release-state}"!="unreleased"]
For detailed information about this release, see the For detailed information about this release, see the
<<release-notes-{elasticsearch_version}, Release notes >> and <<release-notes-{elasticsearch_version}, Release notes >> and
<<breaking-changes-{minor-version}, Breaking changes>>. <<breaking-changes-{minor-version}, Breaking changes>>.
endif::[] endif::[]
// Add previous release to the list // Add previous release to the list
Other versions: Other versions:
{ref-bare}/7.8/release-highlights.html[7.8] {ref-bare}/7.8/release-highlights.html[7.8]
| {ref-bare}/7.7/release-highlights.html[7.7] | {ref-bare}/7.7/release-highlights.html[7.7]
| {ref-bare}/7.6/release-highlights-7.6.0.html[7.6] | {ref-bare}/7.6/release-highlights-7.6.0.html[7.6]
| {ref-bare}/7.5/release-highlights-7.5.0.html[7.5] | {ref-bare}/7.5/release-highlights-7.5.0.html[7.5]
@ -24,17 +24,17 @@ Other versions:
// Use the notable-highlights tag to mark entries that // Use the notable-highlights tag to mark entries that
// should be featured in the Stack Installation and Upgrade Guide: // should be featured in the Stack Installation and Upgrade Guide:
// tag::notable-highlights[] // tag::notable-highlights[]
// [discrete] // [discrete]
// === Heading // === Heading
// //
// Description. // Description.
// end::notable-highlights[] // end::notable-highlights[]
// Omit the notable highlights tag for entries that only need to appear in the ES ref: // Omit the notable highlights tag for entries that only need to appear in the ES ref:
// [float] // [float]
// === Heading // === Heading
// //
// Description. // Description.

24
docs/reference/scripting/using.asciidoc
View File

@ -98,9 +98,11 @@ second version is only compiled once.
If you compile too many unique scripts within a small amount of time, If you compile too many unique scripts within a small amount of time,
Elasticsearch will reject the new dynamic scripts with a Elasticsearch will reject the new dynamic scripts with a
`circuit_breaking_exception` error. By default, up to 15 inline scripts per `circuit_breaking_exception` error. By default, up to 75 scripts per
minute will be compiled. You can change this setting dynamically by setting 5 minutes will be compiled for most contexts and 375 scripts per 5 minutes
`script.max_compilations_rate`. for `ingest` contexts. You can change these settings dynamically by setting
`script.context.$CONTEXT.max_compilations_rate` e.g.,
`script.context.field.max_compilations_rate=100/10m`.
======================================== ========================================
@ -135,7 +137,7 @@ The same script in the normal form:
Scripts may be stored in and retrieved from the cluster state using the Scripts may be stored in and retrieved from the cluster state using the
`_scripts` end-point. `_scripts` end-point.
If the {es} {security-features} are enabled, you must have the following If the {es} {security-features} are enabled, you must have the following
privileges to create, retrieve, and delete stored scripts: privileges to create, retrieve, and delete stored scripts:
* cluster: `all` or `manage` * cluster: `all` or `manage`
@ -241,9 +243,17 @@ templating language]. See <<search-template>> for more information and examples.
All scripts are cached by default so that they only need to be recompiled All scripts are cached by default so that they only need to be recompiled
when updates occur. By default, scripts do not have a time-based expiration, but when updates occur. By default, scripts do not have a time-based expiration, but
you can change this behavior by using the `script.cache.expire` setting. you can configure the size of this cache using the
You can configure the size of this cache by using the `script.cache.max_size` setting. `script.context.$CONTEXT.cache_expire` setting.
By default, the cache size is `100`. By default, the cache size is `100` for all contexts except the `ingest` and the
`processor_conditional` context, where it is `200`.
|====
| Context | Default Cache Size
| `ingest` | 200
| `processor_conditional` | 200
| default | 100
|====
NOTE: The size of scripts is limited to 65,535 bytes. This can be NOTE: The size of scripts is limited to 65,535 bytes. This can be
changed by setting `script.max_size_in_bytes` setting to increase that soft changed by setting `script.max_size_in_bytes` setting to increase that soft

2
qa/remote-clusters/docker-compose-oss.yml
View File

@ -15,7 +15,6 @@ services:
- cluster.routing.allocation.disk.watermark.low=1b - cluster.routing.allocation.disk.watermark.low=1b
- cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.high=1b
- cluster.routing.allocation.disk.watermark.flood_stage=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b
- script.max_compilations_rate=2048/1m
- node.store.allow_mmap=false - node.store.allow_mmap=false
volumes: volumes:
- ./build/oss-repo:/tmp/es-repo - ./build/oss-repo:/tmp/es-repo
@ -50,7 +49,6 @@ services:
- cluster.routing.allocation.disk.watermark.low=1b - cluster.routing.allocation.disk.watermark.low=1b
- cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.high=1b
- cluster.routing.allocation.disk.watermark.flood_stage=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b
- script.max_compilations_rate=2048/1m
- node.store.allow_mmap=false - node.store.allow_mmap=false
volumes: volumes:
- ./build/oss-repo:/tmp/es-repo - ./build/oss-repo:/tmp/es-repo

2
qa/remote-clusters/docker-compose.yml
View File

@ -15,7 +15,6 @@ services:
- cluster.routing.allocation.disk.watermark.low=1b - cluster.routing.allocation.disk.watermark.low=1b
- cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.high=1b
- cluster.routing.allocation.disk.watermark.flood_stage=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b
- script.max_compilations_rate=2048/1m
- node.store.allow_mmap=false - node.store.allow_mmap=false
- xpack.security.enabled=true - xpack.security.enabled=true
- xpack.security.transport.ssl.enabled=true - xpack.security.transport.ssl.enabled=true
@ -65,7 +64,6 @@ services:
- cluster.routing.allocation.disk.watermark.low=1b - cluster.routing.allocation.disk.watermark.low=1b
- cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.high=1b
- cluster.routing.allocation.disk.watermark.flood_stage=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b
- script.max_compilations_rate=2048/1m
- node.store.allow_mmap=false - node.store.allow_mmap=false
- xpack.security.enabled=true - xpack.security.enabled=true
- xpack.security.transport.ssl.enabled=true - xpack.security.transport.ssl.enabled=true

1
server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View File

@ -415,6 +415,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING, ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING,
ScriptService.SCRIPT_CACHE_SIZE_SETTING, ScriptService.SCRIPT_CACHE_SIZE_SETTING,
ScriptService.SCRIPT_CACHE_EXPIRE_SETTING, ScriptService.SCRIPT_CACHE_EXPIRE_SETTING,
ScriptService.SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING,
ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING, ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING,
ScriptService.SCRIPT_MAX_SIZE_IN_BYTES, ScriptService.SCRIPT_MAX_SIZE_IN_BYTES,
ScriptService.TYPES_ALLOWED_SETTING, ScriptService.TYPES_ALLOWED_SETTING,

89
server/src/main/java/org/elasticsearch/script/ScriptCache.java
View File

@ -42,7 +42,7 @@ public class ScriptCache {
private static final Logger logger = LogManager.getLogger(ScriptService.class); private static final Logger logger = LogManager.getLogger(ScriptService.class);
static final Tuple<Integer, TimeValue> UNLIMITED_COMPILATION_RATE = new Tuple<>(0, TimeValue.ZERO); static final CompilationRate UNLIMITED_COMPILATION_RATE = new CompilationRate(0, TimeValue.ZERO);
private final Cache<CacheKey, Object> cache; private final Cache<CacheKey, Object> cache;
private final ScriptMetrics scriptMetrics; private final ScriptMetrics scriptMetrics;
@ -51,14 +51,14 @@ public class ScriptCache {
// Cache settings or derived from settings // Cache settings or derived from settings
final int cacheSize; final int cacheSize;
final TimeValue cacheExpire; final TimeValue cacheExpire;
final Tuple<Integer, TimeValue> rate; final CompilationRate rate;
private final double compilesAllowedPerNano; private final double compilesAllowedPerNano;
private final String contextRateSetting; private final String contextRateSetting;
ScriptCache( ScriptCache(
int cacheMaxSize, int cacheMaxSize,
TimeValue cacheExpire, TimeValue cacheExpire,
Tuple<Integer, TimeValue> maxCompilationRate, CompilationRate maxCompilationRate,
String contextRateSetting String contextRateSetting
) { ) {
this.cacheSize = cacheMaxSize; this.cacheSize = cacheMaxSize;
@ -78,9 +78,9 @@ public class ScriptCache {
this.cache = cacheBuilder.removalListener(new ScriptCacheRemovalListener()).build(); this.cache = cacheBuilder.removalListener(new ScriptCacheRemovalListener()).build();
this.rate = maxCompilationRate; this.rate = maxCompilationRate;
this.compilesAllowedPerNano = ((double) rate.v1()) / rate.v2().nanos(); this.compilesAllowedPerNano = ((double) rate.count) / rate.time.nanos();
this.scriptMetrics = new ScriptMetrics(); this.scriptMetrics = new ScriptMetrics();
this.tokenBucketState = new AtomicReference<TokenBucketState>(new TokenBucketState(this.rate.v1())); this.tokenBucketState = new AtomicReference<TokenBucketState>(new TokenBucketState(this.rate.count));
} }
<FactoryType> FactoryType compile( <FactoryType> FactoryType compile(
@ -156,8 +156,8 @@ public class ScriptCache {
double scriptsPerTimeWindow = current.availableTokens + (timePassed) * compilesAllowedPerNano; double scriptsPerTimeWindow = current.availableTokens + (timePassed) * compilesAllowedPerNano;
// It's been over the time limit anyway, readjust the bucket to be level // It's been over the time limit anyway, readjust the bucket to be level
if (scriptsPerTimeWindow > rate.v1()) { if (scriptsPerTimeWindow > rate.count) {
scriptsPerTimeWindow = rate.v1(); scriptsPerTimeWindow = rate.count;
} }
// If there is enough tokens in the bucket, allow the request and decrease the tokens by 1 // If there is enough tokens in the bucket, allow the request and decrease the tokens by 1
@ -173,7 +173,7 @@ public class ScriptCache {
scriptMetrics.onCompilationLimit(); scriptMetrics.onCompilationLimit();
// Otherwise reject the request // Otherwise reject the request
throw new CircuitBreakingException("[script] Too many dynamic script compilations within, max: [" + throw new CircuitBreakingException("[script] Too many dynamic script compilations within, max: [" +
rate.v1() + "/" + rate.v2() +"]; please use indexed, or scripts with parameters instead; " + rate + "]; please use indexed, or scripts with parameters instead; " +
"this limit can be changed by the [" + contextRateSetting + "] setting", "this limit can be changed by the [" + contextRateSetting + "] setting",
CircuitBreaker.Durability.TRANSIENT); CircuitBreaker.Durability.TRANSIENT);
} }
@ -243,4 +243,77 @@ public class ScriptCache {
this.tokenSuccessfullyTaken = tokenSuccessfullyTaken; this.tokenSuccessfullyTaken = tokenSuccessfullyTaken;
} }
} }
public static class CompilationRate {
public final int count;
public final TimeValue time;
private final String source;
public CompilationRate(Integer count, TimeValue time) {
this.count = count;
this.time = time;
this.source = null;
}
public CompilationRate(Tuple<Integer,TimeValue> rate) {
this(rate.v1(), rate.v2());
}
/**
* Parses a string as a non-negative int count and a {@code TimeValue} as arguments split by a slash
*/
public CompilationRate(String value) {
if (value.contains("/") == false || value.startsWith("/") || value.endsWith("/")) {
throw new IllegalArgumentException("parameter must contain a positive integer and a timevalue, i.e. 10/1m, but was [" +
value + "]");
}
int idx = value.indexOf("/");
String count = value.substring(0, idx);
String time = value.substring(idx + 1);
try {
int rate = Integer.parseInt(count);
if (rate < 0) {
throw new IllegalArgumentException("rate [" + rate + "] must be positive");
}
TimeValue timeValue = TimeValue.parseTimeValue(time, "script.max_compilations_rate");
if (timeValue.nanos() <= 0) {
throw new IllegalArgumentException("time value [" + time + "] must be positive");
}
// protect against a too hard to check limit, like less than a minute
if (timeValue.seconds() < 60) {
throw new IllegalArgumentException("time value [" + time + "] must be at least on a one minute resolution");
}
this.count = rate;
this.time = timeValue;
this.source = value;
} catch (NumberFormatException e) {
// the number format exception message is so confusing, that it makes more sense to wrap it with a useful one
throw new IllegalArgumentException("could not parse [" + count + "] as integer in value [" + value + "]", e);
}
}
public Tuple<Integer,TimeValue> asTuple() {
return new Tuple<>(this.count, this.time);
}
@Override
public String toString() {
return source != null ? source : count + "/" + time.toHumanReadableString(0);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
CompilationRate that = (CompilationRate) o;
return count == that.count &&
Objects.equals(time, that.time);
}
@Override
public int hashCode() {
return Objects.hash(count, time);
}
}
} }

275
server/src/main/java/org/elasticsearch/script/ScriptService.java
View File

@ -34,7 +34,6 @@ import org.elasticsearch.cluster.ClusterStateApplier;
import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Setting.Property;
@ -47,7 +46,6 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
@ -68,51 +66,19 @@ public class ScriptService implements Closeable, ClusterStateApplier {
// Special setting value for SCRIPT_GENERAL_MAX_COMPILATIONS_RATE to indicate the script service should use context // Special setting value for SCRIPT_GENERAL_MAX_COMPILATIONS_RATE to indicate the script service should use context
// specific caches // specific caches
static final Tuple<Integer, TimeValue> USE_CONTEXT_RATE_VALUE = new Tuple<>(-1, TimeValue.MINUS_ONE); static final ScriptCache.CompilationRate USE_CONTEXT_RATE_VALUE = new ScriptCache.CompilationRate(-1, TimeValue.MINUS_ONE);
static final String USE_CONTEXT_RATE_KEY = "use-context"; static final String USE_CONTEXT_RATE_KEY = "use-context";
// a parsing function that requires a non negative int and a timevalue as arguments split by a slash
// this allows you to easily define rates
static final Function<String, Tuple<Integer, TimeValue>> MAX_COMPILATION_RATE_FUNCTION =
(String value) -> {
if (value.contains("/") == false || value.startsWith("/") || value.endsWith("/")) {
throw new IllegalArgumentException("parameter must contain a positive integer and a timevalue, i.e. 10/1m, but was [" +
value + "]");
}
int idx = value.indexOf("/");
String count = value.substring(0, idx);
String time = value.substring(idx + 1);
try {
int rate = Integer.parseInt(count);
if (rate < 0) {
throw new IllegalArgumentException("rate [" + rate + "] must be positive");
}
TimeValue timeValue = TimeValue.parseTimeValue(time, "script.max_compilations_rate");
if (timeValue.nanos() <= 0) {
throw new IllegalArgumentException("time value [" + time + "] must be positive");
}
// protect against a too hard to check limit, like less than a minute
if (timeValue.seconds() < 60) {
throw new IllegalArgumentException("time value [" + time + "] must be at least on a one minute resolution");
}
return Tuple.tuple(rate, timeValue);
} catch (NumberFormatException e) {
// the number format exception message is so confusing, that it makes more sense to wrap it with a useful one
throw new IllegalArgumentException("could not parse [" + count + "] as integer in value [" + value + "]", e);
}
};
public static final Setting<Integer> SCRIPT_GENERAL_CACHE_SIZE_SETTING = public static final Setting<Integer> SCRIPT_GENERAL_CACHE_SIZE_SETTING =
Setting.intSetting("script.cache.max_size", 100, 0, Property.NodeScope); Setting.intSetting("script.cache.max_size", 100, 0, Property.NodeScope, Property.Deprecated);
public static final Setting<TimeValue> SCRIPT_GENERAL_CACHE_EXPIRE_SETTING = public static final Setting<TimeValue> SCRIPT_GENERAL_CACHE_EXPIRE_SETTING =
Setting.positiveTimeSetting("script.cache.expire", TimeValue.timeValueMillis(0), Property.NodeScope); Setting.positiveTimeSetting("script.cache.expire", TimeValue.timeValueMillis(0), Property.NodeScope, Property.Deprecated);
public static final Setting<Integer> SCRIPT_MAX_SIZE_IN_BYTES = public static final Setting<Integer> SCRIPT_MAX_SIZE_IN_BYTES =
Setting.intSetting("script.max_size_in_bytes", 65535, 0, Property.Dynamic, Property.NodeScope); Setting.intSetting("script.max_size_in_bytes", 65535, 0, Property.Dynamic, Property.NodeScope);
public static final Setting<Tuple<Integer, TimeValue>> SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING = public static final Setting<ScriptCache.CompilationRate> SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING =
new Setting<>("script.max_compilations_rate", "75/5m", new Setting<>("script.max_compilations_rate", USE_CONTEXT_RATE_KEY,
(String value) -> value.equals(USE_CONTEXT_RATE_KEY) ? USE_CONTEXT_RATE_VALUE: MAX_COMPILATION_RATE_FUNCTION.apply(value), (String value) -> value.equals(USE_CONTEXT_RATE_KEY) ? USE_CONTEXT_RATE_VALUE: new ScriptCache.CompilationRate(value),
Property.Dynamic, Property.NodeScope); Property.Dynamic, Property.NodeScope, Property.Deprecated);
// Per-context settings // Per-context settings
static final String CONTEXT_PREFIX = "script.context."; static final String CONTEXT_PREFIX = "script.context.";
@ -133,15 +99,18 @@ public class ScriptService implements Closeable, ClusterStateApplier {
// Unlimited compilation rate for context-specific script caches // Unlimited compilation rate for context-specific script caches
static final String UNLIMITED_COMPILATION_RATE_KEY = "unlimited"; static final String UNLIMITED_COMPILATION_RATE_KEY = "unlimited";
public static final Setting.AffixSetting<Tuple<Integer, TimeValue>> SCRIPT_MAX_COMPILATIONS_RATE_SETTING = public static final Setting.AffixSetting<ScriptCache.CompilationRate> SCRIPT_MAX_COMPILATIONS_RATE_SETTING =
Setting.affixKeySetting(CONTEXT_PREFIX, Setting.affixKeySetting(CONTEXT_PREFIX,
"max_compilations_rate", "max_compilations_rate",
key -> new Setting<>(key, "75/5m", key -> new Setting<ScriptCache.CompilationRate>(key, "75/5m",
(String value) -> value.equals(UNLIMITED_COMPILATION_RATE_KEY) ? ScriptCache.UNLIMITED_COMPILATION_RATE: (String value) -> value.equals(UNLIMITED_COMPILATION_RATE_KEY) ? ScriptCache.UNLIMITED_COMPILATION_RATE:
MAX_COMPILATION_RATE_FUNCTION.apply(value), new ScriptCache.CompilationRate(value),
Property.NodeScope, Property.Dynamic)); Property.NodeScope, Property.Dynamic));
private static final Tuple<Integer, TimeValue> SCRIPT_COMPILATION_RATE_ZERO = new Tuple<>(0, TimeValue.ZERO); private static final ScriptCache.CompilationRate SCRIPT_COMPILATION_RATE_ZERO = new ScriptCache.CompilationRate(0, TimeValue.ZERO);
public static final Setting<Boolean> SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING =
Setting.boolSetting("script.disable_max_compilations_rate", false, Property.NodeScope);
public static final String ALLOW_NONE = "none"; public static final String ALLOW_NONE = "none";
@ -160,7 +129,8 @@ public class ScriptService implements Closeable, ClusterStateApplier {
private int maxSizeInBytes; private int maxSizeInBytes;
private final AtomicReference<CacheHolder> cacheHolder; // package private for tests
final AtomicReference<CacheHolder> cacheHolder = new AtomicReference<>();
public ScriptService(Settings settings, Map<String, ScriptEngine> engines, Map<String, ScriptContext<?>> contexts) { public ScriptService(Settings settings, Map<String, ScriptEngine> engines, Map<String, ScriptContext<?>> contexts) {
this.engines = Objects.requireNonNull(engines); this.engines = Objects.requireNonNull(engines);
@ -243,7 +213,7 @@ public class ScriptService implements Closeable, ClusterStateApplier {
// Validation requires knowing which contexts exist. // Validation requires knowing which contexts exist.
this.validateCacheSettings(settings); this.validateCacheSettings(settings);
cacheHolder = new AtomicReference<>(new CacheHolder(settings, contexts.values(), compilationLimitsEnabled())); this.setCacheHolder(settings);
} }
/** /**
@ -259,23 +229,25 @@ public class ScriptService implements Closeable, ClusterStateApplier {
// Handle all updatable per-context settings at once for each context. // Handle all updatable per-context settings at once for each context.
for (ScriptContext<?> context: contexts.values()) { for (ScriptContext<?> context: contexts.values()) {
clusterSettings.addSettingsUpdateConsumer( clusterSettings.addSettingsUpdateConsumer(
(settings) -> cacheHolder.get().updateContextSettings(settings, context), (settings) -> cacheHolder.get().set(context.name, contextCache(settings, context)),
Arrays.asList(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context.name), Arrays.asList(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context.name),
SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context.name), SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context.name),
SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name), SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name),
SCRIPT_GENERAL_CACHE_EXPIRE_SETTING, SCRIPT_GENERAL_CACHE_EXPIRE_SETTING,
// general settings used for fallbacks // general settings used for fallbacks
SCRIPT_GENERAL_CACHE_SIZE_SETTING) SCRIPT_GENERAL_CACHE_SIZE_SETTING
)
); );
} }
// Handle all settings for context and general caches, this flips between general and context caches. // Handle all settings for context and general caches, this flips between general and context caches.
clusterSettings.addSettingsUpdateConsumer( clusterSettings.addSettingsUpdateConsumer(
(settings) -> cacheHolder.set(cacheHolder.get().withUpdatedCacheSettings(settings)), (settings) -> setCacheHolder(settings),
Arrays.asList(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING, Arrays.asList(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING,
SCRIPT_GENERAL_CACHE_EXPIRE_SETTING, SCRIPT_GENERAL_CACHE_EXPIRE_SETTING,
SCRIPT_GENERAL_CACHE_SIZE_SETTING, SCRIPT_GENERAL_CACHE_SIZE_SETTING,
SCRIPT_MAX_COMPILATIONS_RATE_SETTING, SCRIPT_MAX_COMPILATIONS_RATE_SETTING,
SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING,
SCRIPT_CACHE_EXPIRE_SETTING, SCRIPT_CACHE_EXPIRE_SETTING,
SCRIPT_CACHE_SIZE_SETTING), SCRIPT_CACHE_SIZE_SETTING),
this::validateCacheSettings this::validateCacheSettings
@ -290,31 +262,39 @@ public class ScriptService implements Closeable, ClusterStateApplier {
boolean useContext = SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE); boolean useContext = SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE);
List<Setting.AffixSetting<?>> affixes = Arrays.asList(SCRIPT_MAX_COMPILATIONS_RATE_SETTING, SCRIPT_CACHE_EXPIRE_SETTING, List<Setting.AffixSetting<?>> affixes = Arrays.asList(SCRIPT_MAX_COMPILATIONS_RATE_SETTING, SCRIPT_CACHE_EXPIRE_SETTING,
SCRIPT_CACHE_SIZE_SETTING); SCRIPT_CACHE_SIZE_SETTING);
List<String> customRates = new ArrayList<>();
List<String> keys = new ArrayList<>(); List<String> keys = new ArrayList<>();
for (Setting.AffixSetting<?> affix: affixes) { for (Setting.AffixSetting<?> affix: affixes) {
keys.addAll(getConcreteSettingKeys(affix, settings)); for (String context: affix.getAsMap(settings).keySet()) {
String s = affix.getConcreteSettingForNamespace(context).getKey();
if (contexts.containsKey(context) == false) {
throw new IllegalArgumentException("Context [" + context + "] doesn't exist for setting [" + s + "]");
}
keys.add(s);
if (affix.equals(SCRIPT_MAX_COMPILATIONS_RATE_SETTING)) {
customRates.add(s);
}
}
} }
if (useContext == false && keys.isEmpty() == false) { if (useContext == false && keys.isEmpty() == false) {
keys.sort(Comparator.naturalOrder());
throw new IllegalArgumentException("Context cache settings [" + String.join(", ", keys) + "] requires [" + throw new IllegalArgumentException("Context cache settings [" + String.join(", ", keys) + "] requires [" +
SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey() + "] to be [" + USE_CONTEXT_RATE_KEY + "]"); SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey() + "] to be [" + USE_CONTEXT_RATE_KEY + "]");
} }
} if (SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.get(settings)) {
if (customRates.size() > 0) {
/** customRates.sort(Comparator.naturalOrder());
* Get concrete settings keys from affix settings for given Settings. Throws an IllegalArgumentException if the namespace of matching throw new IllegalArgumentException("Cannot set custom context compilation rates [" +
* affix settings do not match any context name. String.join(", ", customRates) + "] if compile rates disabled via [" +
*/ SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.getKey() + "]");
List<String> getConcreteSettingKeys(Setting.AffixSetting<?> setting, Settings settings) { }
List<String> concreteKeys = new ArrayList<>(); if (useContext == false) {
for (String context: setting.getAsMap(settings).keySet()) { throw new IllegalArgumentException("Cannot set custom general compilation rates [" +
String s = setting.getConcreteSettingForNamespace(context).getKey(); SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey() + "] to [" +
if (contexts.containsKey(context) == false) { SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings) + "] if compile rates disabled via [" +
throw new IllegalArgumentException("Context [" + context + "] doesn't exist for setting [" + s + "]"); SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.getKey() + "]");
} }
concreteKeys.add(s);
} }
concreteKeys.sort(Comparator.naturalOrder());
return concreteKeys;
} }
@Override @Override
@ -413,7 +393,7 @@ public class ScriptService implements Closeable, ClusterStateApplier {
return typesAllowed == null || typesAllowed.contains(scriptType.getName()); return typesAllowed == null || typesAllowed.contains(scriptType.getName());
} }
public boolean isContextEnabled(ScriptContext scriptContext) { public boolean isContextEnabled(ScriptContext<?> scriptContext) {
return contextsAllowed == null || contextsAllowed.contains(scriptContext.name); return contextsAllowed == null || contextsAllowed.contains(scriptContext.name);
} }
@ -577,6 +557,67 @@ public class ScriptService implements Closeable, ClusterStateApplier {
clusterState = event.state(); clusterState = event.state();
} }
void setCacheHolder(Settings settings) {
CacheHolder current = cacheHolder.get();
boolean useContext = SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE);
if (current == null) {
if (useContext) {
cacheHolder.set(contextCacheHolder(settings));
} else {
cacheHolder.set(generalCacheHolder(settings));
}
return;
}
// Update
if (useContext) {
if (current.general != null) {
// Flipping to context specific
cacheHolder.set(contextCacheHolder(settings));
}
} else if (current.general == null) {
// Flipping to general
cacheHolder.set(generalCacheHolder(settings));
} else if (current.general.rate.equals(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings)) == false) {
// General compilation rate changed, that setting is the only dynamically updated general setting
cacheHolder.set(generalCacheHolder(settings));
}
}
CacheHolder generalCacheHolder(Settings settings) {
return new CacheHolder(SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(settings), SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(settings),
SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings), SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey());
}
CacheHolder contextCacheHolder(Settings settings) {
Map<String, ScriptCache> contextCache = new HashMap<>(contexts.size());
contexts.forEach((k, v) -> contextCache.put(k, contextCache(settings, v)));
return new CacheHolder(contextCache);
}
ScriptCache contextCache(Settings settings, ScriptContext<?> context) {
Setting<Integer> cacheSizeSetting = SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context.name);
int cacheSize = cacheSizeSetting.existsOrFallbackExists(settings) ? cacheSizeSetting.get(settings) : context.cacheSizeDefault;
Setting<TimeValue> cacheExpireSetting = SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context.name);
TimeValue cacheExpire = cacheExpireSetting.existsOrFallbackExists(settings) ?
cacheExpireSetting.get(settings) : context.cacheExpireDefault;
Setting<ScriptCache.CompilationRate> rateSetting =
SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name);
ScriptCache.CompilationRate rate = null;
if (SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.get(settings) || compilationLimitsEnabled() == false) {
rate = SCRIPT_COMPILATION_RATE_ZERO;
} else if (rateSetting.existsOrFallbackExists(settings)) {
rate = rateSetting.get(settings);
} else {
rate = new ScriptCache.CompilationRate(context.maxCompilationRateDefault);
}
return new ScriptCache(cacheSize, cacheExpire, rate, rateSetting.getKey());
}
/** /**
* Container for the ScriptCache(s). This class operates in two modes: * Container for the ScriptCache(s). This class operates in two modes:
* 1) general mode, if the general script cache is configured. There are no context caches in this case. * 1) general mode, if the general script cache is configured. There are no context caches in this case.
@ -586,80 +627,16 @@ public class ScriptService implements Closeable, ClusterStateApplier {
final ScriptCache general; final ScriptCache general;
final Map<String, AtomicReference<ScriptCache>> contextCache; final Map<String, AtomicReference<ScriptCache>> contextCache;
final Set<ScriptContext<?>> contexts; CacheHolder(int cacheMaxSize, TimeValue cacheExpire, ScriptCache.CompilationRate maxCompilationRate, String contextRateSetting) {
final boolean compilationLimitsEnabled; contextCache = null;
general = new ScriptCache(cacheMaxSize, cacheExpire, maxCompilationRate, contextRateSetting);
CacheHolder(Settings settings, Collection<ScriptContext<?>> contexts, boolean compilationLimitsEnabled) {
this.compilationLimitsEnabled = compilationLimitsEnabled;
this.contexts = Collections.unmodifiableSet(new HashSet<>(contexts));
if (SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE)) {
this.general = null;
Map<String, AtomicReference<ScriptCache>> contextCache = new HashMap<>(this.contexts.size());
for (ScriptContext<?> context : this.contexts) {
contextCache.put(context.name,
new AtomicReference<>(contextFromSettings(settings, context, this.compilationLimitsEnabled)));
}
this.contextCache = Collections.unmodifiableMap(contextCache);
} else {
this.contextCache = null;
this.general = new ScriptCache(
SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(settings),
SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(settings),
compilationLimitsEnabled ?
SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings) :
SCRIPT_COMPILATION_RATE_ZERO,
SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey());
}
} }
/** CacheHolder(Map<String, ScriptCache> context) {
* Create a ScriptCache for the given context. Map<String, AtomicReference<ScriptCache>> refs = new HashMap<>(context.size());
*/ context.forEach((k, v) -> refs.put(k, new AtomicReference<>(v)));
private static ScriptCache contextFromSettings(Settings settings, ScriptContext<?> context, boolean compilationLimitsEnabled) { contextCache = Collections.unmodifiableMap(refs);
String name = context.name; general = null;
Tuple<Integer, TimeValue> compileRate;
Setting<Tuple<Integer, TimeValue>> rateSetting = SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(name);
if (compilationLimitsEnabled == false) {
compileRate = SCRIPT_COMPILATION_RATE_ZERO;
} else if (rateSetting.existsOrFallbackExists(settings)) {
compileRate = rateSetting.get(settings);
} else {
compileRate = context.maxCompilationRateDefault;
}
Setting<TimeValue> cacheExpire = SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(name);
Setting<Integer> cacheSize = SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(name);
return new ScriptCache(cacheSize.existsOrFallbackExists(settings) ? cacheSize.get(settings) : context.cacheSizeDefault,
cacheExpire.existsOrFallbackExists(settings) ? cacheExpire.get(settings) : context.cacheExpireDefault,
compileRate,
SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name).getKey());
}
/**
* Returns a CacheHolder with the given settings. Flips between general and context caches if necessary. Creates new general
* cache if in general cache mode and {@code script.max_compilations_rate} has changed to any value other than {@code use-context}.
*/
CacheHolder withUpdatedCacheSettings(Settings settings) {
if (SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE)) {
if (general != null) {
// Flipping to context specific
logger.debug("Switching to context cache from general cache");
return new CacheHolder(settings, contexts, compilationLimitsEnabled);
}
} else if (general == null) {
// Flipping to general
logger.debug("Switching from context cache to general cache");
return new CacheHolder(settings, contexts, compilationLimitsEnabled);
} else if (general.rate.equals(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings)) == false) {
// General compilation rate changed, that setting is the only dynamically updated general setting
logger.debug("General compilation rate changed from [" + general.rate + "] to [" +
SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings) + "], creating new general cache");
return new CacheHolder(settings, contexts, compilationLimitsEnabled);
}
// no-op change, this is possible when context settings change while in context mode
return this;
} }
/** /**
@ -689,25 +666,25 @@ public class ScriptService implements Closeable, ClusterStateApplier {
return new ScriptCacheStats(general.stats()); return new ScriptCacheStats(general.stats());
} }
Map<String, ScriptStats> context = new HashMap<>(contextCache.size()); Map<String, ScriptStats> context = new HashMap<>(contextCache.size());
for (ScriptContext<?> ctx: contexts) { for (String name: contextCache.keySet()) {
context.put(ctx.name, contextCache.get(ctx.name).get().stats()); context.put(name, contextCache.get(name).get().stats());
} }
return new ScriptCacheStats(context); return new ScriptCacheStats(context);
} }
/** /**
* Update settings for the context cache, if we're in the context cache mode otherwise no-op. * Update a single context cache if we're in the context cache mode otherwise no-op.
*/ */
void updateContextSettings(Settings settings, ScriptContext<?> context) { void set(String name, ScriptCache cache) {
if (general != null) { if (general != null) {
return; return;
} }
AtomicReference<ScriptCache> ref = contextCache.get(context.name); AtomicReference<ScriptCache> ref = contextCache.get(name);
assert ref != null : "expected script cache to exist for context [" + context.name + "]"; assert ref != null : "expected script cache to exist for context [" + name + "]";
ScriptCache cache = ref.get(); ScriptCache oldCache = ref.get();
assert cache != null : "expected script cache to be non-null for context [" + context.name + "]"; assert oldCache != null : "expected script cache to be non-null for context [" + name + "]";
ref.set(contextFromSettings(settings, context, compilationLimitsEnabled)); ref.set(cache);
logger.debug("Replaced context [" + context.name + "] with new settings"); logger.debug("Replaced context [" + name + "] with new settings");
} }
} }
} }

2
server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java
View File

@ -39,7 +39,6 @@ import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
import org.elasticsearch.node.MockNode; import org.elasticsearch.node.MockNode;
import org.elasticsearch.node.Node; import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException; import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.InternalSettingsPlugin;
import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.InternalTestCluster;
@ -69,7 +68,6 @@ public class IndicesServiceCloseTests extends ESTestCase {
.put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo")) .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo"))
.put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent()) .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent())
.put(Node.NODE_NAME_SETTING.getKey(), nodeName) .put(Node.NODE_NAME_SETTING.getKey(), nodeName)
.put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "1000/1m")
.put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created .put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created
.put("transport.type", getTestTransportType()) .put("transport.type", getTestTransportType())
.put(Node.NODE_DATA_SETTING.getKey(), true) .put(Node.NODE_DATA_SETTING.getKey(), true)

14
server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.script; package org.elasticsearch.script;
import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.breaker.CircuitBreakingException;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
@ -30,24 +29,25 @@ public class ScriptCacheTests extends ESTestCase {
public void testCompilationCircuitBreaking() throws Exception { public void testCompilationCircuitBreaking() throws Exception {
final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY); final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY);
final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY); final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY);
Tuple<Integer, TimeValue> rate = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(Settings.EMPTY); ScriptCache.CompilationRate rate = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(Settings.EMPTY);
String settingName = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(); String settingName = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey();
ScriptCache cache = new ScriptCache(size, expire, Tuple.tuple(1, TimeValue.timeValueMinutes(1)), settingName); ScriptCache cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), settingName);
cache.checkCompilationLimit(); // should pass cache.checkCompilationLimit(); // should pass
expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
cache = new ScriptCache(size, expire, (Tuple.tuple(2, TimeValue.timeValueMinutes(1))), settingName); cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), settingName);
cache.checkCompilationLimit(); // should pass cache.checkCompilationLimit(); // should pass
cache.checkCompilationLimit(); // should pass cache.checkCompilationLimit(); // should pass
expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
int count = randomIntBetween(5, 50); int count = randomIntBetween(5, 50);
cache = new ScriptCache(size, expire, (Tuple.tuple(count, TimeValue.timeValueMinutes(1))), settingName); cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), settingName);
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
cache.checkCompilationLimit(); // should pass cache.checkCompilationLimit(); // should pass
} }
expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
cache = new ScriptCache(size, expire, (Tuple.tuple(0, TimeValue.timeValueMinutes(1))), settingName); cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), settingName);
expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit);
cache = new ScriptCache(size, expire, (Tuple.tuple(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1))), settingName); cache = new ScriptCache(size, expire,
new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), settingName);
int largeLimit = randomIntBetween(1000, 10000); int largeLimit = randomIntBetween(1000, 10000);
for (int i = 0; i < largeLimit; i++) { for (int i = 0; i < largeLimit; i++) {
cache.checkCompilationLimit(); cache.checkCompilationLimit();

352
server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
View File

@ -26,8 +26,8 @@ import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.breaker.CircuitBreakingException;
import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentFactory;
@ -38,23 +38,19 @@ import org.junit.Before;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.function.BiFunction; import java.util.function.BiFunction;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors;
import static org.elasticsearch.script.ScriptService.MAX_COMPILATION_RATE_FUNCTION;
import static org.elasticsearch.script.ScriptService.SCRIPT_CACHE_EXPIRE_SETTING; import static org.elasticsearch.script.ScriptService.SCRIPT_CACHE_EXPIRE_SETTING;
import static org.elasticsearch.script.ScriptService.SCRIPT_CACHE_SIZE_SETTING; import static org.elasticsearch.script.ScriptService.SCRIPT_CACHE_SIZE_SETTING;
import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING; import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING;
import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING;
import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING; import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING;
import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING;
import static org.elasticsearch.script.ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING; import static org.elasticsearch.script.ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING;
import static org.elasticsearch.script.ScriptService.USE_CONTEXT_RATE_KEY;
import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.notNullValue;
@ -110,8 +106,8 @@ public class ScriptServiceTests extends ESTestCase {
} }
public void testMaxCompilationRateSetting() throws Exception { public void testMaxCompilationRateSetting() throws Exception {
assertThat(MAX_COMPILATION_RATE_FUNCTION.apply("10/1m"), is(Tuple.tuple(10, TimeValue.timeValueMinutes(1)))); assertThat(new ScriptCache.CompilationRate("10/1m"), is(new ScriptCache.CompilationRate(10, TimeValue.timeValueMinutes(1))));
assertThat(MAX_COMPILATION_RATE_FUNCTION.apply("10/60s"), is(Tuple.tuple(10, TimeValue.timeValueMinutes(1)))); assertThat(new ScriptCache.CompilationRate("10/60s"), is(new ScriptCache.CompilationRate(10, TimeValue.timeValueMinutes(1))));
assertException("10/m", IllegalArgumentException.class, "failed to parse [m]"); assertException("10/m", IllegalArgumentException.class, "failed to parse [m]");
assertException("6/1.6m", IllegalArgumentException.class, "failed to parse [1.6m], fractional time values are not supported"); assertException("6/1.6m", IllegalArgumentException.class, "failed to parse [1.6m], fractional time values are not supported");
assertException("foo/bar", IllegalArgumentException.class, "could not parse [foo] as integer in value [foo/bar]"); assertException("foo/bar", IllegalArgumentException.class, "could not parse [foo] as integer in value [foo/bar]");
@ -131,7 +127,7 @@ public class ScriptServiceTests extends ESTestCase {
} }
private void assertException(String rate, Class<? extends Exception> clazz, String message) { private void assertException(String rate, Class<? extends Exception> clazz, String message) {
Exception e = expectThrows(clazz, () -> MAX_COMPILATION_RATE_FUNCTION.apply(rate)); Exception e = expectThrows(clazz, () -> new ScriptCache.CompilationRate(rate));
assertThat(e.getMessage(), is(message)); assertThat(e.getMessage(), is(message));
} }
@ -233,26 +229,31 @@ public class ScriptServiceTests extends ESTestCase {
} }
public void testCompilationStatsOnCacheHit() throws IOException { public void testCompilationStatsOnCacheHit() throws IOException {
Settings.Builder builder = Settings.builder(); Settings.Builder builder = Settings.builder()
builder.put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1); .put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1)
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "2/1m");
buildScriptService(builder.build()); buildScriptService(builder.build());
Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()); Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap());
ScriptContext<?> context = randomFrom(contexts.values()); ScriptContext<?> context = randomFrom(contexts.values());
scriptService.compile(script, context); scriptService.compile(script, context);
scriptService.compile(script, context); scriptService.compile(script, context);
assertEquals(1L, scriptService.stats().getCompilations()); assertEquals(1L, scriptService.stats().getCompilations());
assertSettingDeprecationsAndWarnings(new Setting<?>[]{SCRIPT_GENERAL_CACHE_SIZE_SETTING,
SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING});
} }
public void testIndexedScriptCountedInCompilationStats() throws IOException { public void testIndexedScriptCountedInCompilationStats() throws IOException {
buildScriptService(Settings.EMPTY); buildScriptService(Settings.EMPTY);
scriptService.compile(new Script(ScriptType.STORED, null, "script", Collections.emptyMap()), randomFrom(contexts.values())); ScriptContext<?> ctx = randomFrom(contexts.values());
scriptService.compile(new Script(ScriptType.STORED, null, "script", Collections.emptyMap()), ctx);
assertEquals(1L, scriptService.stats().getCompilations()); assertEquals(1L, scriptService.stats().getCompilations());
assertEquals(1L, scriptService.cacheStats().getGeneralStats().getCompilations()); assertEquals(1L, scriptService.cacheStats().getContextStats().get(ctx.name).getCompilations());
} }
public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException { public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException {
Settings.Builder builder = Settings.builder(); Settings.Builder builder = Settings.builder();
builder.put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1); builder.put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1);
builder.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "10/1m");
buildScriptService(builder.build()); buildScriptService(builder.build());
scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(contexts.values())); scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(contexts.values()));
scriptService.compile(new Script(ScriptType.INLINE, "test", "2+2", Collections.emptyMap()), randomFrom(contexts.values())); scriptService.compile(new Script(ScriptType.INLINE, "test", "2+2", Collections.emptyMap()), randomFrom(contexts.values()));
@ -260,6 +261,8 @@ public class ScriptServiceTests extends ESTestCase {
assertEquals(2L, scriptService.cacheStats().getGeneralStats().getCompilations()); assertEquals(2L, scriptService.cacheStats().getGeneralStats().getCompilations());
assertEquals(1L, scriptService.stats().getCacheEvictions()); assertEquals(1L, scriptService.stats().getCacheEvictions());
assertEquals(1L, scriptService.cacheStats().getGeneralStats().getCacheEvictions()); assertEquals(1L, scriptService.cacheStats().getGeneralStats().getCacheEvictions());
assertSettingDeprecationsAndWarnings(new Setting<?>[]{SCRIPT_GENERAL_CACHE_SIZE_SETTING,
SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING});
} }
public void testContextCacheStats() throws IOException { public void testContextCacheStats() throws IOException {
@ -273,7 +276,6 @@ public class ScriptServiceTests extends ESTestCase {
".max_compilations_rate] setting" ".max_compilations_rate] setting"
); );
buildScriptService(Settings.builder() buildScriptService(Settings.builder()
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), USE_CONTEXT_RATE_KEY)
.put(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(contextA.name).getKey(), 1) .put(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(contextA.name).getKey(), 1)
.put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(contextA.name).getKey(), aRate) .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(contextA.name).getKey(), aRate)
.put(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(contextB.name).getKey(), 2) .put(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(contextB.name).getKey(), 2)
@ -382,6 +384,7 @@ public class ScriptServiceTests extends ESTestCase {
public void testConflictContextSettings() throws IOException { public void testConflictContextSettings() throws IOException {
IllegalArgumentException illegal = expectThrows(IllegalArgumentException.class, () -> { IllegalArgumentException illegal = expectThrows(IllegalArgumentException.class, () -> {
buildScriptService(Settings.builder() buildScriptService(Settings.builder()
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "10/1m")
.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("field").getKey(), 123).build()); .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("field").getKey(), 123).build());
}); });
assertEquals("Context cache settings [script.context.field.cache_max_size] requires " + assertEquals("Context cache settings [script.context.field.cache_max_size] requires " +
@ -391,6 +394,7 @@ public class ScriptServiceTests extends ESTestCase {
illegal = expectThrows(IllegalArgumentException.class, () -> { illegal = expectThrows(IllegalArgumentException.class, () -> {
buildScriptService(Settings.builder() buildScriptService(Settings.builder()
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "10/1m")
.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("ingest").getKey(), "5m").build()); .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("ingest").getKey(), "5m").build());
}); });
@ -401,6 +405,7 @@ public class ScriptServiceTests extends ESTestCase {
illegal = expectThrows(IllegalArgumentException.class, () -> { illegal = expectThrows(IllegalArgumentException.class, () -> {
buildScriptService(Settings.builder() buildScriptService(Settings.builder()
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "10/1m")
.put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("score").getKey(), "50/5m").build()); .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("score").getKey(), "50/5m").build());
}); });
@ -414,7 +419,8 @@ public class ScriptServiceTests extends ESTestCase {
.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("ingest").getKey(), "5m") .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("ingest").getKey(), "5m")
.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("field").getKey(), 123) .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("field").getKey(), 123)
.put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("score").getKey(), "50/5m") .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("score").getKey(), "50/5m")
.put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY).build()); .build());
assertSettingDeprecationsAndWarnings(new Setting<?>[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING});
} }
public void testFallbackContextSettings() { public void testFallbackContextSettings() {
@ -429,7 +435,7 @@ public class ScriptServiceTests extends ESTestCase {
Settings s = Settings.builder() Settings s = Settings.builder()
.put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), cacheSizeBackup) .put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), cacheSizeBackup)
.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("foo").getKey(), cacheSizeFoo) .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("foo").getKey(), cacheSizeFoo)
.put(ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(), cacheExpireBackup) .put(SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(), cacheExpireBackup)
.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("foo").getKey(), cacheExpireFoo) .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("foo").getKey(), cacheExpireFoo)
.build(); .build();
@ -438,6 +444,7 @@ public class ScriptServiceTests extends ESTestCase {
assertEquals(cacheExpireFooParsed, ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("foo").get(s)); assertEquals(cacheExpireFooParsed, ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("foo").get(s));
assertEquals(cacheExpireBackupParsed, ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("bar").get(s)); assertEquals(cacheExpireBackupParsed, ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("bar").get(s));
assertSettingDeprecationsAndWarnings(new Setting<?>[]{SCRIPT_GENERAL_CACHE_SIZE_SETTING, SCRIPT_GENERAL_CACHE_EXPIRE_SETTING});
} }
public void testUseContextSettingValue() { public void testUseContextSettingValue() {
@ -454,67 +461,42 @@ public class ScriptServiceTests extends ESTestCase {
}); });
assertEquals("parameter must contain a positive integer and a timevalue, i.e. 10/1m, but was [use-context]", illegal.getMessage()); assertEquals("parameter must contain a positive integer and a timevalue, i.e. 10/1m, but was [use-context]", illegal.getMessage());
assertSettingDeprecationsAndWarnings(new Setting<?>[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING});
} }
public void testCacheHolderGeneralConstructor() { public void testCacheHolderGeneralConstructor() throws IOException {
String compilationRate = "77/5m"; String compilationRate = "77/5m";
Tuple<Integer, TimeValue> rate = ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(compilationRate); buildScriptService(
boolean compilationLimitsEnabled = true; Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate).build()
ScriptService.CacheHolder holder = new ScriptService.CacheHolder(
Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate).build(),
Collections.unmodifiableSet(Collections.singleton(newContext("foo"))),
compilationLimitsEnabled
); );
ScriptService.CacheHolder holder = scriptService.cacheHolder.get();
assertNotNull(holder.general); assertNotNull(holder.general);
assertNull(holder.contextCache); assertNull(holder.contextCache);
assertEquals(holder.general.rate, rate); assertEquals(holder.general.rate, new ScriptCache.CompilationRate(compilationRate));
assertSettingDeprecationsAndWarnings(new Setting<?>[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING});
compilationLimitsEnabled = false;
holder = new ScriptService.CacheHolder(
Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate).build(),
new HashSet<>(Collections.singleton(newContext("foo"))),
compilationLimitsEnabled
);
assertNotNull(holder.general);
assertNull(holder.contextCache);
assertEquals(holder.general.rate, new Tuple<>(0, TimeValue.ZERO));
} }
public void testCacheHolderContextConstructor() { public void testCacheHolderContextConstructor() throws IOException {
String fooCompilationRate = "77/5m"; String a = randomFrom(contexts.keySet());
String barCompilationRate = "78/6m"; String b = randomValueOtherThan(a, () -> randomFrom(contexts.keySet()));
boolean compilationLimitsEnabled = true; String aCompilationRate = "77/5m";
String bCompilationRate = "78/6m";
Settings s = Settings.builder() buildScriptService(Settings.builder()
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY) .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(a).getKey(), aCompilationRate)
.put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("foo").getKey(), fooCompilationRate) .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(b).getKey(), bCompilationRate)
.put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("bar").getKey(), barCompilationRate) .build());
.build();
Collection<ScriptContext<?>> contexts = new HashSet<>(Arrays.asList(newContext("foo"), newContext("bar"),
newContext("baz")));
ScriptService.CacheHolder holder = new ScriptService.CacheHolder(s, contexts, compilationLimitsEnabled);
assertNull(holder.general); assertNull(scriptService.cacheHolder.get().general);
assertNotNull(holder.contextCache); assertNotNull(scriptService.cacheHolder.get().contextCache);
assertEquals(3, holder.contextCache.size()); assertEquals(contexts.keySet(), scriptService.cacheHolder.get().contextCache.keySet());
assertEquals(contexts.stream().map(c -> c.name).collect(Collectors.toSet()), holder.contextCache.keySet());
assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(fooCompilationRate), holder.contextCache.get("foo").get().rate); assertEquals(new ScriptCache.CompilationRate(aCompilationRate),
assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(barCompilationRate), holder.contextCache.get("bar").get().rate); scriptService.cacheHolder.get().contextCache.get(a).get().rate);
assertEquals(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getDefault(Settings.EMPTY), assertEquals(new ScriptCache.CompilationRate(bCompilationRate),
holder.contextCache.get("baz").get().rate); scriptService.cacheHolder.get().contextCache.get(b).get().rate);
Tuple<Integer, TimeValue> zero = new Tuple<>(0, TimeValue.ZERO);
compilationLimitsEnabled = false;
holder = new ScriptService.CacheHolder(s, contexts, compilationLimitsEnabled);
assertNotNull(holder.contextCache);
assertEquals(3, holder.contextCache.size());
assertEquals(zero, holder.contextCache.get("foo").get().rate);
assertEquals(zero, holder.contextCache.get("bar").get().rate);
assertEquals(zero, holder.contextCache.get("baz").get().rate);
} }
public void testCompilationRateUnlimitedContextOnly() throws IOException { public void testCompilationRateUnlimitedContextOnly() throws IOException {
@ -533,155 +515,155 @@ public class ScriptServiceTests extends ESTestCase {
ScriptService.UNLIMITED_COMPILATION_RATE_KEY) ScriptService.UNLIMITED_COMPILATION_RATE_KEY)
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY) .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY)
.build()); .build());
assertSettingDeprecationsAndWarnings(new Setting<?>[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING});
} }
public void testCacheHolderChangeSettings() { public void testDisableCompilationRateSetting() throws IOException {
String fooCompilationRate = "77/5m"; IllegalArgumentException illegal = expectThrows(IllegalArgumentException.class, () -> {
String barCompilationRate = "78/6m"; buildScriptService(Settings.builder()
.put("script.context.ingest.max_compilations_rate", "76/10m")
.put("script.context.field.max_compilations_rate", "77/10m")
.put("script.disable_max_compilations_rate", true)
.build());
});
assertEquals("Cannot set custom context compilation rates [script.context.field.max_compilations_rate, " +
"script.context.ingest.max_compilations_rate] if compile rates disabled via " +
"[script.disable_max_compilations_rate]",
illegal.getMessage());
illegal = expectThrows(IllegalArgumentException.class, () -> {
buildScriptService(Settings.builder()
.put("script.disable_max_compilations_rate", true)
.put("script.max_compilations_rate", "76/10m")
.build());
});
assertEquals("Cannot set custom general compilation rates [script.max_compilations_rate] " +
"to [76/10m] if compile rates disabled via [script.disable_max_compilations_rate]",
illegal.getMessage());
buildScriptService(Settings.builder()
.put("script.disable_max_compilations_rate", true)
.build());
assertSettingDeprecationsAndWarnings(new Setting<?>[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING});
}
public void testCacheHolderChangeSettings() throws IOException {
Set<String> contextNames = contexts.keySet();
String a = randomFrom(contextNames);
String aRate = "77/5m";
String b = randomValueOtherThan(a, () -> randomFrom(contextNames));
String bRate = "78/6m";
String c = randomValueOtherThanMany(s -> a.equals(s) || b.equals(s), () -> randomFrom(contextNames));
String compilationRate = "77/5m"; String compilationRate = "77/5m";
Tuple<Integer, TimeValue> generalRate = MAX_COMPILATION_RATE_FUNCTION.apply(compilationRate); ScriptCache.CompilationRate generalRate = new ScriptCache.CompilationRate(compilationRate);
Settings s = Settings.builder() Settings s = Settings.builder()
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate) .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate)
.build(); .build();
Collection<ScriptContext<?>> contexts = new HashSet<>(Arrays.asList(newContext("foo"), newContext("bar"),
newContext("baz"), newContext("qux")));
ScriptService.CacheHolder holder = new ScriptService.CacheHolder(s, contexts, true);
assertNotNull(holder.general); buildScriptService(s);
assertNull(holder.contextCache);
assertEquals(generalRate, holder.general.rate);
holder = holder.withUpdatedCacheSettings(Settings.builder() assertNotNull(scriptService.cacheHolder.get().general);
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY) // Set should not throw when using general cache
.put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("foo").getKey(), fooCompilationRate) scriptService.cacheHolder.get().set(c, scriptService.contextCache(s, contexts.get(c)));
.put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("bar").getKey(), barCompilationRate) assertNull(scriptService.cacheHolder.get().contextCache);
.put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("qux").getKey(), assertEquals(generalRate, scriptService.cacheHolder.get().general.rate);
ScriptService.UNLIMITED_COMPILATION_RATE_KEY)
.build() scriptService.setCacheHolder(Settings.builder()
.put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(a).getKey(), aRate)
.put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(b).getKey(), bRate)
.put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(c).getKey(),
ScriptService.UNLIMITED_COMPILATION_RATE_KEY)
.build()
); );
assertNull(holder.general); assertNull(scriptService.cacheHolder.get().general);
assertNotNull(holder.contextCache); assertNotNull(scriptService.cacheHolder.get().contextCache);
assertEquals(4, holder.contextCache.size()); // get of missing context should be null
assertEquals(contexts.stream().map(c -> c.name).collect(Collectors.toSet()), holder.contextCache.keySet()); assertNull(scriptService.cacheHolder.get().get(
randomValueOtherThanMany(contexts.keySet()::contains, () -> randomAlphaOfLength(8)))
assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(fooCompilationRate), holder.contextCache.get("foo").get().rate);
assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(barCompilationRate), holder.contextCache.get("bar").get().rate);
assertEquals(ScriptCache.UNLIMITED_COMPILATION_RATE, holder.contextCache.get("qux").get().rate);
assertEquals(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getDefault(Settings.EMPTY),
holder.contextCache.get("baz").get().rate);
holder.updateContextSettings(Settings.builder()
.put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("bar").getKey(), fooCompilationRate).build(),
newContext("bar")
); );
assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(fooCompilationRate), holder.contextCache.get("bar").get().rate); assertEquals(contexts.keySet(), scriptService.cacheHolder.get().contextCache.keySet());
holder = holder.withUpdatedCacheSettings(s); String d = randomValueOtherThanMany(Arrays.asList(a, b, c)::contains, () -> randomFrom(contextNames));
assertNotNull(holder.general); assertEquals(new ScriptCache.CompilationRate(aRate),
assertNull(holder.contextCache); scriptService.cacheHolder.get().contextCache.get(a).get().rate);
assertEquals(generalRate, holder.general.rate); assertEquals(new ScriptCache.CompilationRate(bRate),
scriptService.cacheHolder.get().contextCache.get(b).get().rate);
assertEquals(ScriptCache.UNLIMITED_COMPILATION_RATE,
scriptService.cacheHolder.get().contextCache.get(c).get().rate);
holder = holder.withUpdatedCacheSettings( scriptService.cacheHolder.get().set(b, scriptService.contextCache(Settings.builder()
Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), barCompilationRate).build() .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(b).getKey(), aRate).build(),
contexts.get(b)));
assertEquals(new ScriptCache.CompilationRate(aRate),
scriptService.cacheHolder.get().contextCache.get(b).get().rate);
scriptService.setCacheHolder(s);
assertNotNull(scriptService.cacheHolder.get().general);
assertNull(scriptService.cacheHolder.get().contextCache);
assertEquals(generalRate, scriptService.cacheHolder.get().general.rate);
scriptService.setCacheHolder(
Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), bRate).build()
); );
assertNotNull(holder.general); assertNotNull(scriptService.cacheHolder.get().general);
assertNull(holder.contextCache); assertNull(scriptService.cacheHolder.get().contextCache);
assertEquals(MAX_COMPILATION_RATE_FUNCTION.apply(barCompilationRate), holder.general.rate); assertEquals(new ScriptCache.CompilationRate(bRate), scriptService.cacheHolder.get().general.rate);
ScriptService.CacheHolder update = holder.withUpdatedCacheSettings( ScriptService.CacheHolder holder = scriptService.cacheHolder.get();
Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), barCompilationRate).build() scriptService.setCacheHolder(
Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), bRate).build()
); );
assertSame(holder, update); assertEquals(holder, scriptService.cacheHolder.get());
assertSettingDeprecationsAndWarnings(new Setting<?>[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING});
} }
public void testFallbackToContextDefaults() { public void testFallbackToContextDefaults() throws IOException {
Tuple<Integer, TimeValue> contextDefaultRate = new Tuple<>(randomIntBetween(10, 1024),
TimeValue.timeValueMinutes(randomIntBetween(10, 200)));
String name = "foo";
ScriptContext<?> foo = new ScriptContext<>(name,
ScriptContextTests.DummyScript.Factory.class,
randomIntBetween(1, 1024),
TimeValue.timeValueMinutes(randomIntBetween(10, 200)),
contextDefaultRate);
int generalCacheSize = randomIntBetween(1, 1024);
TimeValue generalExpire = TimeValue.timeValueMinutes(randomIntBetween(10, 200));
String contextRateStr = randomIntBetween(10, 1024) + "/" + randomIntBetween(10, 200) + "m"; String contextRateStr = randomIntBetween(10, 1024) + "/" + randomIntBetween(10, 200) + "m";
Tuple<Integer, TimeValue> contextRate = MAX_COMPILATION_RATE_FUNCTION.apply(contextRateStr); ScriptCache.CompilationRate contextRate = new ScriptCache.CompilationRate(contextRateStr);
int contextCacheSize = randomIntBetween(1, 1024); int contextCacheSize = randomIntBetween(1, 1024);
TimeValue contextExpire = TimeValue.timeValueMinutes(randomIntBetween(10, 200)); TimeValue contextExpire = TimeValue.timeValueMinutes(randomIntBetween(10, 200));
buildScriptService(
Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "75/5m").build()
);
String name = "ingest";
// Use context specific // Use context specific
ScriptService.CacheHolder contextCache = new ScriptService.CacheHolder( scriptService.setCacheHolder(Settings.builder()
Settings.builder() .put(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextCacheSize)
.put(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextCacheSize) .put(SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextExpire)
.put(SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextExpire) .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextRateStr)
.put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextRateStr) .build()
.put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), generalCacheSize) );
.put(SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(), generalExpire)
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), USE_CONTEXT_RATE_KEY)
.build(),
Arrays.asList(foo),
true);
assertNotNull(contextCache.contextCache); ScriptService.CacheHolder holder = scriptService.cacheHolder.get();
assertNotNull(contextCache.contextCache.get(name)); assertNotNull(holder.contextCache);
assertNotNull(contextCache.contextCache.get(name).get()); assertNotNull(holder.contextCache.get(name));
assertNotNull(holder.contextCache.get(name).get());
assertEquals(contextRate, contextCache.contextCache.get(name).get().rate); assertEquals(contextRate, holder.contextCache.get(name).get().rate);
assertEquals(contextCacheSize, contextCache.contextCache.get(name).get().cacheSize); assertEquals(contextCacheSize, holder.contextCache.get(name).get().cacheSize);
assertEquals(contextExpire, contextCache.contextCache.get(name).get().cacheExpire); assertEquals(contextExpire, holder.contextCache.get(name).get().cacheExpire);
// Fallback to general
contextCache = new ScriptService.CacheHolder(
Settings.builder()
.put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), generalCacheSize)
.put(SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(), generalExpire)
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), USE_CONTEXT_RATE_KEY)
.build(),
Arrays.asList(foo),
true);
assertNotNull(contextCache.contextCache);
assertNotNull(contextCache.contextCache.get(name));
assertNotNull(contextCache.contextCache.get(name).get());
assertEquals(contextDefaultRate, contextCache.contextCache.get(name).get().rate);
assertEquals(generalCacheSize, contextCache.contextCache.get(name).get().cacheSize);
assertEquals(generalExpire, contextCache.contextCache.get(name).get().cacheExpire);
ScriptContext<?> ingest = contexts.get(name);
// Fallback to context defaults // Fallback to context defaults
contextCache = new ScriptService.CacheHolder( buildScriptService(Settings.EMPTY);
Settings.builder()
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), USE_CONTEXT_RATE_KEY)
.build(),
Arrays.asList(foo),
true);
assertNotNull(contextCache.contextCache); holder = scriptService.cacheHolder.get();
assertNotNull(contextCache.contextCache.get(name)); assertNotNull(holder.contextCache);
assertNotNull(contextCache.contextCache.get(name).get()); assertNotNull(holder.contextCache.get(name));
assertNotNull(holder.contextCache.get(name).get());
assertEquals(contextDefaultRate, contextCache.contextCache.get(name).get().rate); assertEquals(ingest.maxCompilationRateDefault, holder.contextCache.get(name).get().rate.asTuple());
assertEquals(foo.cacheSizeDefault, contextCache.contextCache.get(name).get().cacheSize); assertEquals(ingest.cacheSizeDefault, holder.contextCache.get(name).get().cacheSize);
assertEquals(foo.cacheExpireDefault, contextCache.contextCache.get(name).get().cacheExpire); assertEquals(ingest.cacheExpireDefault, holder.contextCache.get(name).get().cacheExpire);
// Use context specific for ingest assertSettingDeprecationsAndWarnings(new Setting<?>[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING});
contextCache = new ScriptService.CacheHolder(
Settings.builder()
.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), USE_CONTEXT_RATE_KEY)
.build(),
Arrays.asList(foo, IngestScript.CONTEXT, IngestConditionalScript.CONTEXT),
true);
assertEquals(new Tuple<>(375, TimeValue.timeValueMinutes(5)),
contextCache.contextCache.get("ingest").get().rate);
assertEquals(200, contextCache.contextCache.get("ingest").get().cacheSize);
assertEquals(TimeValue.timeValueMillis(0), contextCache.contextCache.get("ingest").get().cacheExpire);
} }
private void assertCompileRejected(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) { private void assertCompileRejected(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) {
@ -700,8 +682,4 @@ public class ScriptServiceTests extends ESTestCase {
notNullValue() notNullValue()
); );
} }
ScriptContext<ScriptContextTests.DummyScript.Factory> newContext(String name) {
return new ScriptContext<>(name, ScriptContextTests.DummyScript.Factory.class);
}
} }

10
test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
View File

@ -106,6 +106,7 @@ import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeService; import org.elasticsearch.node.NodeService;
import org.elasticsearch.node.NodeValidationException; import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.ScriptModule;
import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.SearchService; import org.elasticsearch.search.SearchService;
import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.tasks.TaskManager;
@ -520,11 +521,14 @@ public final class InternalTestCluster extends TestCluster {
} }
if (random.nextBoolean()) { if (random.nextBoolean()) {
builder.put(ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 0, 2000)); String ctx = randomFrom(random, ScriptModule.CORE_CONTEXTS.keySet());
builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(ctx).getKey(),
RandomNumbers.randomIntBetween(random, 0, 2000));
} }
if (random.nextBoolean()) { if (random.nextBoolean()) {
builder.put(ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(), String ctx = randomFrom(random, ScriptModule.CORE_CONTEXTS.keySet());
timeValueMillis(RandomNumbers.randomIntBetween(random, 750, 10000000)).getStringRep()); builder.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(ctx).getKey(),
timeValueMillis(RandomNumbers.randomIntBetween(random, 750, 10000000)).getStringRep());
} }
if (random.nextBoolean()) { if (random.nextBoolean()) {
int initialMillisBound = RandomNumbers.randomIntBetween(random,10, 100); int initialMillisBound = RandomNumbers.randomIntBetween(random,10, 100);

2
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
View File

@ -61,7 +61,6 @@ import org.elasticsearch.license.LicenseService;
import org.elasticsearch.license.LicensesMetadata; import org.elasticsearch.license.LicensesMetadata;
import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.snapshots.RestoreInfo; import org.elasticsearch.snapshots.RestoreInfo;
import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.snapshots.RestoreService;
@ -220,7 +219,6 @@ public abstract class CcrIntegTestCase extends ESTestCase {
builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b"); builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b");
builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b"); builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b");
builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b"); builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b");
builder.put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "2048/1m");
// wait short time for other active shards before actually deleting, default 30s not needed in tests // wait short time for other active shards before actually deleting, default 30s not needed in tests
builder.put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS)); builder.put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS));
builder.putList(DISCOVERY_SEED_HOSTS_SETTING.getKey()); // empty list disables a port scan for other nodes builder.putList(DISCOVERY_SEED_HOSTS_SETTING.getKey()); // empty list disables a port scan for other nodes

3
x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
View File

@ -51,6 +51,9 @@ public class DeprecationChecks {
(settings, pluginsAndModules) -> NodeDeprecationChecks.checkThreadPoolListenerSize(settings), (settings, pluginsAndModules) -> NodeDeprecationChecks.checkThreadPoolListenerSize(settings),
NodeDeprecationChecks::checkClusterRemoteConnectSetting, NodeDeprecationChecks::checkClusterRemoteConnectSetting,
NodeDeprecationChecks::checkNodeLocalStorageSetting, NodeDeprecationChecks::checkNodeLocalStorageSetting,
NodeDeprecationChecks::checkGeneralScriptSizeSetting,
NodeDeprecationChecks::checkGeneralScriptExpireSetting,
NodeDeprecationChecks::checkGeneralScriptCompileSettings,
(settings, pluginsAndModules) -> NodeDeprecationChecks.checkNodeBasicLicenseFeatureEnabledSetting(settings, (settings, pluginsAndModules) -> NodeDeprecationChecks.checkNodeBasicLicenseFeatureEnabledSetting(settings,
XPackSettings.ENRICH_ENABLED_SETTING), XPackSettings.ENRICH_ENABLED_SETTING),
(settings, pluginsAndModules) -> NodeDeprecationChecks.checkNodeBasicLicenseFeatureEnabledSetting(settings, (settings, pluginsAndModules) -> NodeDeprecationChecks.checkNodeBasicLicenseFeatureEnabledSetting(settings,

64
x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java
View File

@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.env.Environment; import org.elasticsearch.env.Environment;
import org.elasticsearch.node.Node; import org.elasticsearch.node.Node;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.FixedExecutorBuilder;
import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.transport.RemoteClusterService;
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
@ -145,6 +146,39 @@ class NodeDeprecationChecks {
); );
} }
public static DeprecationIssue checkGeneralScriptSizeSetting(final Settings settings, final PluginsAndModules pluginsAndModules) {
return checkDeprecatedSetting(
settings,
pluginsAndModules,
ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING,
ScriptService.SCRIPT_CACHE_SIZE_SETTING,
"a script context",
"https://www.elastic.co/guide/en/elasticsearch/reference/7.9/breaking-changes-7.9.html#deprecate_general_script_cache_size"
);
}
public static DeprecationIssue checkGeneralScriptExpireSetting(final Settings settings, final PluginsAndModules pluginsAndModules) {
return checkDeprecatedSetting(
settings,
pluginsAndModules,
ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING,
ScriptService.SCRIPT_CACHE_EXPIRE_SETTING,
"a script context",
"https://www.elastic.co/guide/en/elasticsearch/reference/7.9/breaking-changes-7.9.html#deprecate_general_script_expire"
);
}
public static DeprecationIssue checkGeneralScriptCompileSettings(final Settings settings, final PluginsAndModules pluginsAndModules) {
return checkDeprecatedSetting(
settings,
pluginsAndModules,
ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING,
ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING,
"a script context",
"https://www.elastic.co/guide/en/elasticsearch/reference/7.9/breaking-changes-7.9.html#deprecate_general_script_compile_rate"
);
}
private static DeprecationIssue checkDeprecatedSetting( private static DeprecationIssue checkDeprecatedSetting(
final Settings settings, final Settings settings,
final PluginsAndModules pluginsAndModules, final PluginsAndModules pluginsAndModules,
@ -173,6 +207,36 @@ class NodeDeprecationChecks {
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details); return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details);
} }
private static DeprecationIssue checkDeprecatedSetting(
final Settings settings,
final PluginsAndModules pluginsAndModules,
final Setting<?> deprecatedSetting,
final Setting.AffixSetting<?> replacementSetting,
final String star,
final String url) {
assert deprecatedSetting.isDeprecated() : deprecatedSetting;
if (deprecatedSetting.exists(settings) == false) {
return null;
}
final String deprecatedSettingKey = deprecatedSetting.getKey();
final String replacementSettingKey = replacementSetting.getKey();
final String value = deprecatedSetting.get(settings).toString();
final String message = String.format(
Locale.ROOT,
"setting [%s] is deprecated in favor of grouped setting [%s]",
deprecatedSettingKey,
replacementSettingKey);
final String details = String.format(
Locale.ROOT,
"the setting [%s] is currently set to [%s], instead set [%s] to [%s] where * is %s",
deprecatedSettingKey,
value,
replacementSettingKey,
value,
star);
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details);
}
static DeprecationIssue checkRemovedSetting(final Settings settings, final Setting<?> removedSetting, final String url) { static DeprecationIssue checkRemovedSetting(final Settings settings, final Setting<?> removedSetting, final String url) {
if (removedSetting.exists(settings) == false) { if (removedSetting.exists(settings) == false) {
return null; return null;

50
x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java
View File

@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.env.Environment; import org.elasticsearch.env.Environment;
import org.elasticsearch.node.Node; import org.elasticsearch.node.Node;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.transport.RemoteClusterService;
import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.XPackSettings;
@ -181,6 +182,54 @@ public class NodeDeprecationChecksTests extends ESTestCase {
assertSettingDeprecationsAndWarnings(new String[]{"thread_pool.listener.size"}); assertSettingDeprecationsAndWarnings(new String[]{"thread_pool.listener.size"});
} }
public void testGeneralScriptSizeSetting() {
final int size = randomIntBetween(1, 4);
final Settings settings = Settings.builder().put("script.cache.max_size", size).build();
final PluginsAndModules pluginsAndModules = new PluginsAndModules(Collections.emptyList(), Collections.emptyList());
final List<DeprecationIssue> issues =
DeprecationChecks.filterChecks(DeprecationChecks.NODE_SETTINGS_CHECKS, c -> c.apply(settings, pluginsAndModules));
final DeprecationIssue expected = new DeprecationIssue(
DeprecationIssue.Level.CRITICAL,
"setting [script.cache.max_size] is deprecated in favor of grouped setting [script.context.*.cache_max_size]",
"https://www.elastic.co/guide/en/elasticsearch/reference/7.9/breaking-changes-7.9.html#deprecate_general_script_cache_size",
"the setting [script.cache.max_size] is currently set to [" + size + "], instead set [script.context.*.cache_max_size] " +
"to [" + size + "] where * is a script context");
assertThat(issues, contains(expected));
assertSettingDeprecationsAndWarnings(new Setting<?>[]{ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING});
}
public void testGeneralScriptExpireSetting() {
final String expire = randomIntBetween(1, 4) + "m";
final Settings settings = Settings.builder().put("script.cache.expire", expire).build();
final PluginsAndModules pluginsAndModules = new PluginsAndModules(Collections.emptyList(), Collections.emptyList());
final List<DeprecationIssue> issues =
DeprecationChecks.filterChecks(DeprecationChecks.NODE_SETTINGS_CHECKS, c -> c.apply(settings, pluginsAndModules));
final DeprecationIssue expected = new DeprecationIssue(
DeprecationIssue.Level.CRITICAL,
"setting [script.cache.expire] is deprecated in favor of grouped setting [script.context.*.cache_expire]",
"https://www.elastic.co/guide/en/elasticsearch/reference/7.9/breaking-changes-7.9.html#deprecate_general_script_expire",
"the setting [script.cache.expire] is currently set to [" + expire + "], instead set [script.context.*.cache_expire] to " +
"[" + expire + "] where * is a script context");
assertThat(issues, contains(expected));
assertSettingDeprecationsAndWarnings(new Setting<?>[]{ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING});
}
public void testGeneralScriptCompileSettings() {
final String rate = randomIntBetween(1, 100) + "/" + randomIntBetween(1, 200) + "m";
final Settings settings = Settings.builder().put("script.max_compilations_rate", rate).build();
final PluginsAndModules pluginsAndModules = new PluginsAndModules(Collections.emptyList(), Collections.emptyList());
final List<DeprecationIssue> issues =
DeprecationChecks.filterChecks(DeprecationChecks.NODE_SETTINGS_CHECKS, c -> c.apply(settings, pluginsAndModules));
final DeprecationIssue expected = new DeprecationIssue(
DeprecationIssue.Level.CRITICAL,
"setting [script.max_compilations_rate] is deprecated in favor of grouped setting [script.context.*.max_compilations_rate]",
"https://www.elastic.co/guide/en/elasticsearch/reference/7.9/breaking-changes-7.9.html#deprecate_general_script_compile_rate",
"the setting [script.max_compilations_rate] is currently set to [" + rate +
"], instead set [script.context.*.max_compilations_rate] to [" + rate + "] where * is a script context");
assertThat(issues, contains(expected));
assertSettingDeprecationsAndWarnings(new Setting<?>[]{ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING});
}
public void testClusterRemoteConnectSetting() { public void testClusterRemoteConnectSetting() {
final boolean value = randomBoolean(); final boolean value = randomBoolean();
final Settings settings = Settings.builder().put(RemoteClusterService.ENABLE_REMOTE_CLUSTERS.getKey(), value).build(); final Settings settings = Settings.builder().put(RemoteClusterService.ENABLE_REMOTE_CLUSTERS.getKey(), value).build();
@ -272,5 +321,4 @@ public class NodeDeprecationChecksTests extends ESTestCase {
equalTo("the setting [node.removed_setting] is currently set to [value], remove this setting")); equalTo("the setting [node.removed_setting] is currently set to [value], remove this setting"));
assertThat(issue.getUrl(), equalTo("https://removed-setting.example.com")); assertThat(issue.getUrl(), equalTo("https://removed-setting.example.com"));
} }
} }