honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add a soft limit to the field name length (#40309) 

Adds an optional limit to the length of field names, throws an IllegalArgumentException if the limit is breached. 
Closes #33651
This commit is contained in:
alex101101 2019-03-27 00:29:24 +08:00 committed by Christoph Büscher
parent f2b5960f90
commit fb8ad0cf30
5 changed files with 143 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/reference/mapping.asciidoc
View File

@ -97,6 +97,12 @@ causing a mapping explosion:
100 objects within a nested field, will actually create 101 documents, as
each nested object will be indexed as a separate hidden document.
`index.mapping.field_name_length.limit`::
Setting for the maximum length of a field name. The default value is
Long.MAX_VALUE (no limit). This setting isn't really something that addresses
mappings explosion but might still be useful if you want to limit the field length.
It usually shouldn't be necessary to set this setting. The default is okay
unless a user starts to add a huge number of fields with really long names.
[float]
== Dynamic mapping

1
server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
View File

@ -154,6 +154,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING,
MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING,
MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING,
BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
IndexModule.INDEX_STORE_TYPE_SETTING,
IndexModule.INDEX_STORE_PRE_LOAD_SETTING,

24
server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -69,6 +69,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Stream;
import static java.util.Collections.emptyMap;
import static java.util.Collections.unmodifiableMap;
@ -101,7 +102,9 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
public static final Setting<Long> INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING =
Setting.longSetting("index.mapping.total_fields.limit", 1000L, 0, Property.Dynamic, Property.IndexScope);
public static final Setting<Long> INDEX_MAPPING_DEPTH_LIMIT_SETTING =
Setting.longSetting("index.mapping.depth.limit", 20L, 1, Property.Dynamic, Property.IndexScope);
Setting.longSetting("index.mapping.depth.limit", 20L, 1, Property.Dynamic, Property.IndexScope);
public static final Setting<Long> INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING =
Setting.longSetting("index.mapping.field_name_length.limit", Long.MAX_VALUE, 1L, Property.Dynamic, Property.IndexScope);
public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true;
@Deprecated
public static final Setting<Boolean> INDEX_MAPPER_DYNAMIC_SETTING =
@ -503,6 +506,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
// Also, don't take metadata mappers into account for the field limit check
checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size() - metadataMappers.length
+ fieldAliasMappers.size() );
checkFieldNameSoftLimit(objectMappers, fieldMappers, fieldAliasMappers);
}
results.put(newMapper.type(), newMapper);
@ -623,6 +627,24 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
}
}
private void checkFieldNameSoftLimit(Collection<ObjectMapper> objectMappers,
Collection<FieldMapper> fieldMappers,
Collection<FieldAliasMapper> fieldAliasMappers) {
final long maxFieldNameLength = indexSettings.getValue(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING);
Stream.of(objectMappers.stream(), fieldMappers.stream(), fieldAliasMappers.stream())
.reduce(Stream::concat)
.orElseGet(Stream::empty)
.forEach(mapper -> {
String name = mapper.simpleName();
if (name.length() > maxFieldNameLength) {
throw new IllegalArgumentException("Field name [" + name + "] in index [" + index().getName() +
"] is too long. The limit is set to [" + maxFieldNameLength + "] characters but was ["
+ name.length() + "] characters");
}
});
}
private void checkPartitionedIndexConstraints(DocumentMapper newMapper) {
if (indexSettings.getIndexMetaData().isRoutingPartitionedIndex()) {
if (!newMapper.routingFieldMapper().required()) {

111
server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
View File

@ -323,4 +323,115 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
+ " that indices can have at most one type.", e.getMessage());
}
public void testFieldNameLengthLimit() throws Throwable {
int maxFieldNameLength = randomIntBetween(15, 20);
String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a");
Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength)
.build();
MapperService mapperService = createIndex("test1", settings).mapperService();
CompressedXContent mapping = new CompressedXContent(BytesReference.bytes(
XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field")
.field("type", "text")
.endObject()
.endObject()
.endObject().endObject()));
mapperService.merge("type", mapping, MergeReason.MAPPING_UPDATE);
CompressedXContent mappingUpdate = new CompressedXContent(BytesReference.bytes(
XContentFactory.jsonBuilder().startObject()
.startObject("properties")
.startObject(testString)
.field("type", "text")
.endObject()
.endObject()
.endObject()));
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
mapperService.merge("type", mappingUpdate, MergeReason.MAPPING_UPDATE);
});
assertEquals("Field name [" + testString + "] in index [test1] is too long. " +
"The limit is set to [" + maxFieldNameLength + "] characters but was ["
+ testString.length() + "] characters", e.getMessage());
}
public void testObjectNameLengthLimit() throws Throwable {
int maxFieldNameLength = randomIntBetween(15, 20);
String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a");
Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength)
.build();
MapperService mapperService = createIndex("test1", settings).mapperService();
CompressedXContent mapping = new CompressedXContent(BytesReference.bytes(
XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject(testString)
.field("type", "object")
.endObject()
.endObject()
.endObject().endObject()));
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
mapperService.merge("type", mapping, MergeReason.MAPPING_UPDATE);
});
assertEquals("Field name [" + testString + "] in index [test1] is too long. " +
"The limit is set to [" + maxFieldNameLength + "] characters but was ["
+ testString.length() + "] characters", e.getMessage());
}
public void testAliasFieldNameLengthLimit() throws Throwable {
int maxFieldNameLength = randomIntBetween(15, 20);
String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a");
Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength)
.build();
MapperService mapperService = createIndex("test1", settings).mapperService();
CompressedXContent mapping = new CompressedXContent(BytesReference.bytes(
XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject(testString)
.field("type", "alias")
.field("path", "field")
.endObject()
.startObject("field")
.field("type", "text")
.endObject()
.endObject()
.endObject().endObject()));
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
mapperService.merge("type", mapping, MergeReason.MAPPING_UPDATE);
});
assertEquals("Field name [" + testString + "] in index [test1] is too long. " +
"The limit is set to [" + maxFieldNameLength + "] characters but was ["
+ testString.length() + "] characters", e.getMessage());
}
public void testMappingRecoverySkipFieldNameLengthLimit() throws Throwable {
int maxFieldNameLength = randomIntBetween(15, 20);
String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a");
Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength)
.build();
MapperService mapperService = createIndex("test1", settings).mapperService();
CompressedXContent mapping = new CompressedXContent(BytesReference.bytes(
XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject(testString)
.field("type", "text")
.endObject()
.endObject()
.endObject().endObject()));
DocumentMapper documentMapper = mapperService.merge("type", mapping, MergeReason.MAPPING_RECOVERY);
assertEquals(testString, documentMapper.mappers().getMapper(testString).simpleName());
}
}

3
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
View File

@ -218,7 +218,7 @@ public class TransportResumeFollowActionTests extends ESTestCase {
validate(request, leaderIMD, followIMD, UUIDs, mapperService);
}
}
public void testDynamicIndexSettingsAreClassified() {
// We should be conscious which dynamic settings are replicated from leader to follower index.
// This is the list of settings that should be replicated:
@ -230,6 +230,7 @@ public class TransportResumeFollowActionTests extends ESTestCase {
replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPER_DYNAMIC_SETTING);
replicatedSettings.add(IndexSettings.MAX_NGRAM_DIFF_SETTING);
replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING);