honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add per-index setting to limit number of nested fields 

Closes #14983
This commit is contained in:
Yannick Welsch 2016-01-14 15:05:00 +01:00
parent b2baf039fd
commit a1b8dd2de9
5 changed files with 112 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -95,6 +95,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
public static final String DEFAULT_MAPPING = "_default_";
public static final String INDEX_MAPPER_DYNAMIC_SETTING = "index.mapper.dynamic";
public static final String INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING = "index.mapping.nested_fields.limit";
public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true;
private static ObjectHashSet<String> META_FIELDS = ObjectHashSet.from(
"_uid", "_id", "_type", "_all", "_parent", "_routing", "_index",
@ -242,12 +243,12 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
// only apply the default mapping if we don't have the type yet
&& mappers.containsKey(type) == false;
DocumentMapper mergeWith = parse(type, mappingSource, applyDefault);
return merge(mergeWith, updateAllTypes);
return merge(mergeWith, reason, updateAllTypes);
}
}
}
private synchronized DocumentMapper merge(DocumentMapper mapper, boolean updateAllTypes) {
private synchronized DocumentMapper merge(DocumentMapper mapper, MergeReason reason, boolean updateAllTypes) {
if (mapper.type().length() == 0) {
throw new InvalidTypeNameException("mapping type name is empty");
}
@ -300,6 +301,11 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
}
}
fullPathObjectMappers = Collections.unmodifiableMap(fullPathObjectMappers);
if (reason == MergeReason.MAPPING_UPDATE) {
checkNestedFieldsLimit(fullPathObjectMappers);
}
Set<String> parentTypes = this.parentTypes;
if (oldMapper == null && newMapper.parentFieldMapper().active()) {
parentTypes = new HashSet<>(parentTypes.size() + 1);
@ -412,6 +418,19 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
}
}
private void checkNestedFieldsLimit(Map<String, ObjectMapper> fullPathObjectMappers) {
long allowedNestedFields = indexSettings.getSettings().getAsLong(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, 50L);
long actualNestedFields = 0;
for (ObjectMapper objectMapper : fullPathObjectMappers.values()) {
if (objectMapper.nested().isNested()) {
actualNestedFields++;
}
}
if (allowedNestedFields >= 0 && actualNestedFields > allowedNestedFields) {
throw new IllegalArgumentException("Limit of nested fields [" + allowedNestedFields + "] in index [" + index().name() + "] has been exceeded");
}
}
public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException {
String defaultMappingSource;
if (PercolatorService.TYPE_NAME.equals(mappingType)) {

64
core/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java
View File

@ -20,14 +20,22 @@
package org.elasticsearch.index.mapper.nested;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MapperService.MergeReason;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.mapper.object.ObjectMapper.Dynamic;
import org.elasticsearch.test.ESSingleNodeTestCase;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.function.Function;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
@ -340,4 +348,58 @@ public class NestedMappingTests extends ESSingleNodeTestCase {
assertThat(doc.docs().get(1).get("field"), nullValue());
assertThat(doc.docs().get(2).get("field"), equalTo("value"));
}
}
public void testLimitOfNestedFieldsPerIndex() throws Exception {
Function<String, String> mapping = type -> {
try {
return XContentFactory.jsonBuilder().startObject().startObject(type).startObject("properties")
.startObject("nested1").field("type", "nested").startObject("properties")
.startObject("nested2").field("type", "nested")
.endObject().endObject()
.endObject().endObject().endObject().string();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
};
// default limit allows at least two nested fields
createIndex("test1").mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
// explicitly setting limit to 0 prevents nested fields
try {
createIndex("test2", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, 0).build())
.mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsString("Limit of nested fields [0] in index [test2] has been exceeded"));
}
// setting limit to 1 with 2 nested fields fails
try {
createIndex("test3", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, 1).build())
.mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsString("Limit of nested fields [1] in index [test3] has been exceeded"));
}
MapperService mapperService = createIndex("test4", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, 2)
.build()).mapperService();
mapperService.merge("type1", new CompressedXContent(mapping.apply("type1")), MergeReason.MAPPING_UPDATE, false);
// merging same fields, but different type is ok
mapperService.merge("type2", new CompressedXContent(mapping.apply("type2")), MergeReason.MAPPING_UPDATE, false);
// adding new fields from different type is not ok
String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type3").startObject("properties").startObject("nested3")
.field("type", "nested").startObject("properties").endObject().endObject().endObject().endObject().string();
try {
mapperService.merge("type3", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE, false);
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsString("Limit of nested fields [2] in index [test4] has been exceeded"));
}
// do not check nested fields limit if mapping is not updated
createIndex("test5", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, 0).build())
.mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_RECOVERY, false);
}
}

7
docs/reference/mapping/types/nested.asciidoc
View File

@ -199,3 +199,10 @@ phase. Instead, highlighting needs to be performed via
=============================================
==== Limiting the number of `nested` fields
Indexing a document with 100 nested fields actually indexes 101 documents as each nested
document is indexed as a separate document. To safeguard against ill-defined mappings
the number of nested fields that can be defined per index has been limited to 50. This
default limit can be changed with the index setting `index.mapping.nested_fields.limit`.

2
docs/reference/migration/index.asciidoc
View File

@ -18,6 +18,8 @@ See <<setup-upgrade>> for more info.
--
include::migrate_3_0.asciidoc[]
include::migrate_2_3.asciidoc[]
include::migrate_2_2.asciidoc[]
include::migrate_2_1.asciidoc[]

19
docs/reference/migration/migrate_2_3.asciidoc Normal file
View File

@ -0,0 +1,19 @@
[[breaking-changes-2.3]]
== Breaking changes in 2.3
This section discusses the changes that you need to be aware of when migrating
your application to Elasticsearch 2.3.
* <<breaking_23_index_apis>>
[[breaking_23_index_apis]]
=== Mappings
==== Limit to the number of `nested` fields
Indexing a document with 100 nested fields actually indexes 101 documents as each nested
document is indexed as a separate document. To safeguard against ill-defined mappings
the number of nested fields that can be defined per index has been limited to 50.
This default limit can be changed with the index setting `index.mapping.nested_fields.limit`.
Note that the limit is only checked when new indices are created or mappings are updated. It
will thus only affect existing pre-2.3 indices if their mapping is changed.