honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add soft limit on allowed number of script fields in request (#26598) 

Requesting to many script_fields in a search request can be costly
because of script execution. This change introduces a soft limit on the number
of script fields that are allowed per request. The setting can be
changed per index using the index.max_script_fields setting.

Relates to #26390
This commit is contained in:
Christoph Büscher 2017-09-13 17:22:16 +02:00 committed by GitHub
parent 64770b3fbd
commit 027c555c9b
7 changed files with 131 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
View File

@ -112,6 +112,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
IndexSettings.MAX_RESULT_WINDOW_SETTING,
IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING,
IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING,
IndexSettings.MAX_SCRIPT_FIELDS_SETTING,
IndexSettings.MAX_RESCORE_WINDOW_SETTING,
IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING,
IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,

23
core/src/main/java/org/elasticsearch/index/IndexSettings.java
View File

@ -98,6 +98,15 @@ public final class IndexSettings {
*/
public static final Setting<Integer> MAX_INNER_RESULT_WINDOW_SETTING =
Setting.intSetting("index.max_inner_result_window", 100, 1, Property.Dynamic, Property.IndexScope);
/**
* Index setting describing the maximum value of allowed `script_fields`that can be retrieved
* per search request. The default maximum of 50 is defensive for the reason that retrieving
* script fields is a costly operation.
*/
public static final Setting<Integer> MAX_SCRIPT_FIELDS_SETTING =
Setting.intSetting("index.max_script_fields", 32, 0, Property.Dynamic, Property.IndexScope);
/**
* Index setting describing the maximum value of allowed `docvalue_fields`that can be retrieved
* per search request. The default maximum of 100 is defensive for the reason that retrieving
@ -229,6 +238,7 @@ public final class IndexSettings {
private volatile int maxAdjacencyMatrixFilters;
private volatile int maxRescoreWindow;
private volatile int maxDocvalueFields;
private volatile int maxScriptFields;
private volatile boolean TTLPurgeDisabled;
/**
* The maximum number of refresh listeners allows on this shard.
@ -331,6 +341,7 @@ public final class IndexSettings {
maxAdjacencyMatrixFilters = scopedSettings.get(MAX_ADJACENCY_MATRIX_FILTERS_SETTING);
maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING);
maxDocvalueFields = scopedSettings.get(MAX_DOCVALUE_FIELDS_SEARCH_SETTING);
maxScriptFields = scopedSettings.get(MAX_SCRIPT_FIELDS_SETTING);
TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING);
maxRefreshListeners = scopedSettings.get(MAX_REFRESH_LISTENERS_PER_SHARD);
maxSlicesPerScroll = scopedSettings.get(MAX_SLICES_PER_SCROLL);
@ -361,6 +372,7 @@ public final class IndexSettings {
scopedSettings.addSettingsUpdateConsumer(MAX_ADJACENCY_MATRIX_FILTERS_SETTING, this::setMaxAdjacencyMatrixFilters);
scopedSettings.addSettingsUpdateConsumer(MAX_RESCORE_WINDOW_SETTING, this::setMaxRescoreWindow);
scopedSettings.addSettingsUpdateConsumer(MAX_DOCVALUE_FIELDS_SEARCH_SETTING, this::setMaxDocvalueFields);
scopedSettings.addSettingsUpdateConsumer(MAX_SCRIPT_FIELDS_SETTING, this::setMaxScriptFields);
scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer);
scopedSettings.addSettingsUpdateConsumer(INDEX_GC_DELETES_SETTING, this::setGCDeletes);
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, this::setTranslogFlushThresholdSize);
@ -628,6 +640,17 @@ public final class IndexSettings {
this.maxDocvalueFields = maxDocvalueFields;
}
/**
* Returns the maximum number of allowed script_fields to retrieve in a search request
*/
public int getMaxScriptFields() {
return this.maxScriptFields;
}
private void setMaxScriptFields(int maxScriptFields) {
this.maxScriptFields = maxScriptFields;
}
/**
* Returns the GC deletes cycle in milliseconds.
*/

7
core/src/main/java/org/elasticsearch/search/SearchService.java
View File

@ -788,6 +788,13 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
}
}
if (source.scriptFields() != null) {
int maxAllowedScriptFields = context.mapperService().getIndexSettings().getMaxScriptFields();
if (source.scriptFields().size() > maxAllowedScriptFields) {
throw new IllegalArgumentException(
"Trying to retrieve too many script_fields. Must be less than or equal to: [" + maxAllowedScriptFields
+ "] but was [" + source.scriptFields().size() + "]. This limit can be set by changing the ["
+ IndexSettings.MAX_SCRIPT_FIELDS_SETTING.getKey() + "] index level setting.");
}
for (org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField field : source.scriptFields()) {
SearchScript.Factory factory = scriptService.compile(field.script(), SearchScript.CONTEXT);
SearchScript.LeafFactory searchScript = factory.newFactory(field.script().getParams(), context.lookup());

16
core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
View File

@ -326,6 +326,22 @@ public class IndexSettingsTests extends ESTestCase {
assertEquals(IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxDocvalueFields());
}
public void testMaxScriptFields() {
IndexMetaData metaData = newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexSettings.MAX_SCRIPT_FIELDS_SETTING.getKey(), 100).build());
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
assertEquals(100, settings.getMaxScriptFields());
settings.updateIndexMetaData(
newIndexMeta("index", Settings.builder().put(IndexSettings.MAX_SCRIPT_FIELDS_SETTING.getKey(), 20).build()));
assertEquals(20, settings.getMaxScriptFields());
settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY));
assertEquals(IndexSettings.MAX_SCRIPT_FIELDS_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxScriptFields());
metaData = newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build());
settings = new IndexSettings(metaData, Settings.EMPTY);
assertEquals(IndexSettings.MAX_SCRIPT_FIELDS_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxScriptFields());
}
public void testMaxAdjacencyMatrixFiltersSetting() {
IndexMetaData metaData = newIndexMeta("index", Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)

54
core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
View File

@ -47,6 +47,10 @@ import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.SearchPlugin;
import org.elasticsearch.script.MockScriptEngine;
import org.elasticsearch.script.MockScriptPlugin;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.support.ValueType;
@ -60,11 +64,14 @@ import org.elasticsearch.test.ESSingleNodeTestCase;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import static java.util.Collections.singletonList;
import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@ -83,7 +90,19 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
@Override
protected Collection<Class<? extends Plugin>> getPlugins() {
return pluginList(FailOnRewriteQueryPlugin.class);
return pluginList(FailOnRewriteQueryPlugin.class, CustomScriptPlugin.class);
}
public static class CustomScriptPlugin extends MockScriptPlugin {
static final String DUMMY_SCRIPT = "dummyScript";
@Override
protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
return Collections.singletonMap(DUMMY_SCRIPT, vars -> {
return "dummy";
});
}
}
@Override
@ -290,6 +309,39 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
}
}
/**
* test that getting more than the allowed number of script_fields throws an exception
*/
public void testMaxScriptFieldsSearch() throws IOException {
createIndex("index");
final SearchService service = getInstanceFromNode(SearchService.class);
final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
final IndexShard indexShard = indexService.getShard(0);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// adding the maximum allowed number of script_fields to retrieve
int maxScriptFields = indexService.getIndexSettings().getMaxScriptFields();
for (int i = 0; i < maxScriptFields; i++) {
searchSourceBuilder.scriptField("field" + i,
new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
}
try (SearchContext context = service.createContext(new ShardSearchLocalRequest(indexShard.shardId(), 1, SearchType.DEFAULT,
searchSourceBuilder, new String[0], false, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f), null)) {
assertNotNull(context);
searchSourceBuilder.scriptField("anotherScriptField",
new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap()));
IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
() -> service.createContext(new ShardSearchLocalRequest(indexShard.shardId(), 1, SearchType.DEFAULT,
searchSourceBuilder, new String[0], false, new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f), null));
assertEquals(
"Trying to retrieve too many script_fields. Must be less than or equal to: [" + maxScriptFields + "] but was ["
+ (maxScriptFields + 1)
+ "]. This limit can be set by changing the [index.max_script_fields] index level setting.",
ex.getMessage());
}
}
public static class FailOnRewriteQueryPlugin extends Plugin implements SearchPlugin {
@Override
public List<QuerySpec<?>> getQueries() {

5
docs/reference/index-modules.asciidoc
View File

@ -139,6 +139,11 @@ specific index module:
Defaults to `100`. Doc-value fields are costly since they might incur
a per-field per-document seek.
`index.max_script_fields`::
The maximum number of `script_fields` that are allowed in a query.
Defaults to `32`.
`index.blocks.read_only`::
Set to `true` to make the index and index metadata read only, `false` to

26
rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml
View File

@ -69,3 +69,29 @@ setup:
query:
match_all: {}
docvalue_fields: ["one", "two", "three"]
---
"Script_fields size limit":
- skip:
version: " - 6.99.99"
reason: soft limit for script_fields only available as of 7.0.0
- do:
indices.create:
index: test_2
body:
settings:
index.max_script_fields: 2
- do:
catch: /Trying to retrieve too many script_fields\. Must be less than or equal to[:] \[2\] but was \[3\]\. This limit can be set by changing the \[index.max_script_fields\] index level setting\./
search:
index: test_2
body:
query:
match_all: {}
script_fields: {
"test1" : { "script" : { "lang": "painless", "source": "1" }},
"test2" : { "script" : { "lang": "painless", "source": "1" }},
"test3" : { "script" : { "lang": "painless", "source": "1" }}
}