From 361adcf3870457e87d3fc94e9c799ffea2382ec8 Mon Sep 17 00:00:00 2001 From: Yanjun Huang Date: Sat, 26 Mar 2016 23:37:42 -0700 Subject: [PATCH] Add limit to total number of fields in mapping. #17357 This is to prevent mapping explosion when dynamic keys such as UUID are used as field names. index.mapping.total_fields.limit specifies the total number of fields an index can have. An exception will be thrown when the limit is reached. The default limit is 1000. Value 0 means no limit. This setting is runtime adjustable Closes #11443 --- .../common/settings/IndexScopedSettings.java | 1 + .../index/mapper/MapperService.java | 12 ++++++++- .../cluster/SimpleClusterStateIT.java | 5 +++- .../index/mapper/MapperServiceTests.java | 26 +++++++++++++++++++ .../mapping/dynamic/field-mapping.asciidoc | 6 +++++ 5 files changed, 48 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index fb498283d7b..37a071a7cf3 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -128,6 +128,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { PercolatorQueryCache.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING, MapperService.INDEX_MAPPER_DYNAMIC_SETTING, MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, + MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, IndexModule.INDEX_STORE_TYPE_SETTING, IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING, diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 73b94e60b46..f46586ccb1a 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -84,6 +84,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable { public static final String DEFAULT_MAPPING = "_default_"; public static final Setting INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING = Setting.longSetting("index.mapping.nested_fields.limit", 50L, 0, Property.Dynamic, Property.IndexScope); + public static final Setting INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING = + Setting.longSetting("index.mapping.total_fields.limit", 1000L, 0, Property.Dynamic, Property.IndexScope); public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true; public static final Setting INDEX_MAPPER_DYNAMIC_SETTING = Setting.boolSetting("index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT, Property.IndexScope); @@ -289,6 +291,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable { // deserializing cluster state that was sent by the master node, // this check will be skipped. checkNestedFieldsLimit(fullPathObjectMappers); + checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size()); } Set parentTypes = this.parentTypes; @@ -403,11 +406,18 @@ public class MapperService extends AbstractIndexComponent implements Closeable { actualNestedFields++; } } - if (allowedNestedFields >= 0 && actualNestedFields > allowedNestedFields) { + if (actualNestedFields > allowedNestedFields) { throw new IllegalArgumentException("Limit of nested fields [" + allowedNestedFields + "] in index [" + index().getName() + "] has been exceeded"); } } + private void checkTotalFieldsLimit(long totalMappers) { + long allowedTotalFields = indexSettings.getValue(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING); + if (allowedTotalFields < totalMappers) { + throw new IllegalArgumentException("Limit of total fields [" + allowedTotalFields + "] in index [" + index().getName() + "] has been exceeded"); + } + } + public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException { String defaultMappingSource; if (PercolatorFieldMapper.TYPE_NAME.equals(mappingType)) { diff --git a/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java index 6916cfc4e31..da5f3149d6c 100644 --- a/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.CollectionAssertions; import org.junit.Before; @@ -145,7 +146,9 @@ public class SimpleClusterStateIT extends ESIntegTestCase { int numberOfShards = scaledRandomIntBetween(1, cluster().numDataNodes()); // if the create index is ack'ed, then all nodes have successfully processed the cluster state assertAcked(client().admin().indices().prepareCreate("test") - .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards, + IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0, + MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), Long.MAX_VALUE) .addMapping("type", mapping) .setTimeout("60s").get()); ensureGreen(); // wait for green state, so its both green, and there are no more pending events diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 58244ce7ad0..bf7f8e0b5cc 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -30,15 +30,21 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Rule; import org.junit.rules.ExpectedException; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.function.Function; import java.util.concurrent.ExecutionException; import static org.hamcrest.CoreMatchers.containsString; @@ -135,4 +141,24 @@ public class MapperServiceTests extends ESSingleNodeTestCase { assertFalse(indexService.mapperService().hasMapping(MapperService.DEFAULT_MAPPING)); } + public void testTotalFieldsExceedsLimit() throws Throwable { + Function mapping = type -> { + try { + return XContentFactory.jsonBuilder().startObject().startObject(type).startObject("properties") + .startObject("field1").field("type", "string") + .endObject().endObject().endObject().endObject().string(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + createIndex("test1").mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false); + //set total number of fields to 1 to trigger an exception + try { + createIndex("test2", Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1).build()) + .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("Limit of total fields [1] in index [test2] has been exceeded")); + } + } } diff --git a/docs/reference/mapping/dynamic/field-mapping.asciidoc b/docs/reference/mapping/dynamic/field-mapping.asciidoc index f8612958f9c..238052a9739 100644 --- a/docs/reference/mapping/dynamic/field-mapping.asciidoc +++ b/docs/reference/mapping/dynamic/field-mapping.asciidoc @@ -30,6 +30,12 @@ detected. All other datatypes must be mapped explicitly. Besides the options listed below, dynamic field mapping rules can be further customised with <>. +[[total-fields-limit]] +==== Total fields limit + +To avoid mapping explosion, Index has a default limit of 1000 total number of fields. +The default setting can be updated with `index.mapping.total_fields.limit`. + [[date-detection]] ==== Date detection