diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc index b6a7052f69a..2e09a0a8ca2 100644 --- a/docs/reference/mapping.asciidoc +++ b/docs/reference/mapping.asciidoc @@ -97,6 +97,12 @@ causing a mapping explosion: 100 objects within a nested field, will actually create 101 documents, as each nested object will be indexed as a separate hidden document. +`index.mapping.field_name_length.limit`:: + Setting for the maximum length of a field name. The default value is + Long.MAX_VALUE (no limit). This setting isn't really something that addresses + mappings explosion but might still be useful if you want to limit the field length. + It usually shouldn't be necessary to set this setting. The default is okay + unless a user starts to add a huge number of fields with really long names. [float] == Dynamic mapping diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 01d9be7b5ea..cb3dfa8e598 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -154,6 +154,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING, + MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, IndexModule.INDEX_STORE_TYPE_SETTING, IndexModule.INDEX_STORE_PRE_LOAD_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 53d49e657a3..171f8c4bb8b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -69,6 +69,7 @@ import java.util.Objects; import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Stream; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; @@ -101,7 +102,9 @@ public class MapperService extends AbstractIndexComponent implements Closeable { public static final Setting INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING = Setting.longSetting("index.mapping.total_fields.limit", 1000L, 0, Property.Dynamic, Property.IndexScope); public static final Setting INDEX_MAPPING_DEPTH_LIMIT_SETTING = - Setting.longSetting("index.mapping.depth.limit", 20L, 1, Property.Dynamic, Property.IndexScope); + Setting.longSetting("index.mapping.depth.limit", 20L, 1, Property.Dynamic, Property.IndexScope); + public static final Setting INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING = + Setting.longSetting("index.mapping.field_name_length.limit", Long.MAX_VALUE, 1L, Property.Dynamic, Property.IndexScope); public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true; @Deprecated public static final Setting INDEX_MAPPER_DYNAMIC_SETTING = @@ -503,6 +506,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable { // Also, don't take metadata mappers into account for the field limit check checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size() - metadataMappers.length + fieldAliasMappers.size() ); + checkFieldNameSoftLimit(objectMappers, fieldMappers, fieldAliasMappers); } results.put(newMapper.type(), newMapper); @@ -623,6 +627,24 @@ public class MapperService extends AbstractIndexComponent implements Closeable { } } + private void checkFieldNameSoftLimit(Collection objectMappers, + Collection fieldMappers, + Collection fieldAliasMappers) { + final long maxFieldNameLength = indexSettings.getValue(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); + + Stream.of(objectMappers.stream(), fieldMappers.stream(), fieldAliasMappers.stream()) + .reduce(Stream::concat) + .orElseGet(Stream::empty) + .forEach(mapper -> { + String name = mapper.simpleName(); + if (name.length() > maxFieldNameLength) { + throw new IllegalArgumentException("Field name [" + name + "] in index [" + index().getName() + + "] is too long. The limit is set to [" + maxFieldNameLength + "] characters but was [" + + name.length() + "] characters"); + } + }); + } + private void checkPartitionedIndexConstraints(DocumentMapper newMapper) { if (indexSettings.getIndexMetaData().isRoutingPartitionedIndex()) { if (!newMapper.routingFieldMapper().required()) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 2c1a75b40d4..d8c120e492d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -323,4 +323,115 @@ public class MapperServiceTests extends ESSingleNodeTestCase { + " that indices can have at most one type.", e.getMessage()); } + public void testFieldNameLengthLimit() throws Throwable { + int maxFieldNameLength = randomIntBetween(15, 20); + String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); + Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) + .build(); + MapperService mapperService = createIndex("test1", settings).mapperService(); + + CompressedXContent mapping = new CompressedXContent(BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "text") + .endObject() + .endObject() + .endObject().endObject())); + + mapperService.merge("type", mapping, MergeReason.MAPPING_UPDATE); + + CompressedXContent mappingUpdate = new CompressedXContent(BytesReference.bytes( + XContentFactory.jsonBuilder().startObject() + .startObject("properties") + .startObject(testString) + .field("type", "text") + .endObject() + .endObject() + .endObject())); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + mapperService.merge("type", mappingUpdate, MergeReason.MAPPING_UPDATE); + }); + + assertEquals("Field name [" + testString + "] in index [test1] is too long. " + + "The limit is set to [" + maxFieldNameLength + "] characters but was [" + + testString.length() + "] characters", e.getMessage()); + } + + public void testObjectNameLengthLimit() throws Throwable { + int maxFieldNameLength = randomIntBetween(15, 20); + String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); + Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) + .build(); + MapperService mapperService = createIndex("test1", settings).mapperService(); + + CompressedXContent mapping = new CompressedXContent(BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject(testString) + .field("type", "object") + .endObject() + .endObject() + .endObject().endObject())); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + mapperService.merge("type", mapping, MergeReason.MAPPING_UPDATE); + }); + + assertEquals("Field name [" + testString + "] in index [test1] is too long. " + + "The limit is set to [" + maxFieldNameLength + "] characters but was [" + + testString.length() + "] characters", e.getMessage()); + } + + public void testAliasFieldNameLengthLimit() throws Throwable { + int maxFieldNameLength = randomIntBetween(15, 20); + String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); + Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) + .build(); + MapperService mapperService = createIndex("test1", settings).mapperService(); + + CompressedXContent mapping = new CompressedXContent(BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject(testString) + .field("type", "alias") + .field("path", "field") + .endObject() + .startObject("field") + .field("type", "text") + .endObject() + .endObject() + .endObject().endObject())); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + mapperService.merge("type", mapping, MergeReason.MAPPING_UPDATE); + }); + + assertEquals("Field name [" + testString + "] in index [test1] is too long. " + + "The limit is set to [" + maxFieldNameLength + "] characters but was [" + + testString.length() + "] characters", e.getMessage()); + } + + public void testMappingRecoverySkipFieldNameLengthLimit() throws Throwable { + int maxFieldNameLength = randomIntBetween(15, 20); + String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); + Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) + .build(); + MapperService mapperService = createIndex("test1", settings).mapperService(); + + CompressedXContent mapping = new CompressedXContent(BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject(testString) + .field("type", "text") + .endObject() + .endObject() + .endObject().endObject())); + + DocumentMapper documentMapper = mapperService.merge("type", mapping, MergeReason.MAPPING_RECOVERY); + + assertEquals(testString, documentMapper.mappers().getMapper(testString).simpleName()); + } + } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 57bc30210fa..2167f56b8fc 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -218,7 +218,7 @@ public class TransportResumeFollowActionTests extends ESTestCase { validate(request, leaderIMD, followIMD, UUIDs, mapperService); } } - + public void testDynamicIndexSettingsAreClassified() { // We should be conscious which dynamic settings are replicated from leader to follower index. // This is the list of settings that should be replicated: @@ -230,6 +230,7 @@ public class TransportResumeFollowActionTests extends ESTestCase { replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING); + replicatedSettings.add(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPER_DYNAMIC_SETTING); replicatedSettings.add(IndexSettings.MAX_NGRAM_DIFF_SETTING); replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING);