honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add limit to total number of fields in mapping. #17357 

This is to prevent mapping explosion when dynamic keys such as UUID are used as field names. index.mapping.total_fields.limit specifies the total number of fields an index can have. An exception will be thrown when the limit is reached. The default limit is 1000. Value 0 means no limit. This setting is runtime adjustable

Closes #11443
This commit is contained in:
Yanjun Huang 2016-03-26 23:37:42 -07:00 committed by Adrien Grand
parent c356b30cff
commit 361adcf387
5 changed files with 48 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
View File

@ -128,6 +128,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
PercolatorQueryCache.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING,
MapperService.INDEX_MAPPER_DYNAMIC_SETTING,
MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING,
MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
IndexModule.INDEX_STORE_TYPE_SETTING,
IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING,

12
core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -84,6 +84,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
public static final String DEFAULT_MAPPING = "_default_";
public static final Setting<Long> INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING =
Setting.longSetting("index.mapping.nested_fields.limit", 50L, 0, Property.Dynamic, Property.IndexScope);
public static final Setting<Long> INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING =
Setting.longSetting("index.mapping.total_fields.limit", 1000L, 0, Property.Dynamic, Property.IndexScope);
public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true;
public static final Setting<Boolean> INDEX_MAPPER_DYNAMIC_SETTING =
Setting.boolSetting("index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT, Property.IndexScope);
@ -289,6 +291,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
// deserializing cluster state that was sent by the master node,
// this check will be skipped.
checkNestedFieldsLimit(fullPathObjectMappers);
checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size());
}
Set<String> parentTypes = this.parentTypes;
@ -403,11 +406,18 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
actualNestedFields++;
}
}
if (allowedNestedFields >= 0 && actualNestedFields > allowedNestedFields) {
if (actualNestedFields > allowedNestedFields) {
throw new IllegalArgumentException("Limit of nested fields [" + allowedNestedFields + "] in index [" + index().getName() + "] has been exceeded");
}
}
private void checkTotalFieldsLimit(long totalMappers) {
long allowedTotalFields = indexSettings.getValue(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
if (allowedTotalFields < totalMappers) {
throw new IllegalArgumentException("Limit of total fields [" + allowedTotalFields + "] in index [" + index().getName() + "] has been exceeded");
}
}
public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException {
String defaultMappingSource;
if (PercolatorFieldMapper.TYPE_NAME.equals(mappingType)) {

5
core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.hamcrest.CollectionAssertions;
import org.junit.Before;
@ -145,7 +146,9 @@ public class SimpleClusterStateIT extends ESIntegTestCase {
int numberOfShards = scaledRandomIntBetween(1, cluster().numDataNodes());
// if the create index is ack'ed, then all nodes have successfully processed the cluster state
assertAcked(client().admin().indices().prepareCreate("test")
.setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards,
IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0,
MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), Long.MAX_VALUE)
.addMapping("type", mapping)
.setTimeout("60s").get());
ensureGreen(); // wait for green state, so its both green, and there are no more pending events

26
core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
View File

@ -30,15 +30,21 @@ import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.MapperService.MergeReason;
import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.junit.Rule;
import org.junit.rules.ExpectedException;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.function.Function;
import java.util.concurrent.ExecutionException;
import static org.hamcrest.CoreMatchers.containsString;
@ -135,4 +141,24 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
assertFalse(indexService.mapperService().hasMapping(MapperService.DEFAULT_MAPPING));
}
public void testTotalFieldsExceedsLimit() throws Throwable {
Function<String, String> mapping = type -> {
try {
return XContentFactory.jsonBuilder().startObject().startObject(type).startObject("properties")
.startObject("field1").field("type", "string")
.endObject().endObject().endObject().endObject().string();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
};
createIndex("test1").mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
//set total number of fields to 1 to trigger an exception
try {
createIndex("test2", Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1).build())
.mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsString("Limit of total fields [1] in index [test2] has been exceeded"));
}
}
}

6
docs/reference/mapping/dynamic/field-mapping.asciidoc
View File

@ -30,6 +30,12 @@ detected. All other datatypes must be mapped explicitly.
Besides the options listed below, dynamic field mapping rules can be further
customised with <<dynamic-templates,`dynamic_templates`>>.
[[total-fields-limit]]
==== Total fields limit
To avoid mapping explosion, Index has a default limit of 1000 total number of fields.
The default setting can be updated with `index.mapping.total_fields.limit`.
[[date-detection]]
==== Date detection