honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Check if root mapping is actually valid 

When a mapping is declared and the type is known from the uri
then the type can be skipped in the body (see #4483). However,
there was no check if the given keys actually make a valid mapping.

closes #5864
closes #6093
This commit is contained in:
Britta Weber 2014-05-10 18:23:07 +02:00
parent caacce9429
commit 4b2e4becc7
14 changed files with 238 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Nullable;
@ -48,7 +49,9 @@ import org.elasticsearch.index.mapper.object.RootObjectMapper;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.similarity.SimilarityLookupService;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.index.mapper.MapperBuilders.doc;
@ -201,31 +204,38 @@ public class DocumentMapperParser extends AbstractIndexComponent {
Mapper.TypeParser.ParserContext parserContext = parserContext();
// parse RootObjectMapper
DocumentMapper.Builder docBuilder = doc(index.name(), indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext));
for (Map.Entry<String, Object> entry : mapping.entrySet()) {
Iterator<Map.Entry<String, Object>> iterator = mapping.entrySet().iterator();
// parse DocumentMapper
while(iterator.hasNext()) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if ("index_analyzer".equals(fieldName)) {
iterator.remove();
NamedAnalyzer analyzer = analysisService.analyzer(fieldNode.toString());
if (analyzer == null) {
throw new MapperParsingException("Analyzer [" + fieldNode.toString() + "] not found for index_analyzer setting on root type [" + type + "]");
}
docBuilder.indexAnalyzer(analyzer);
} else if ("search_analyzer".equals(fieldName)) {
iterator.remove();
NamedAnalyzer analyzer = analysisService.analyzer(fieldNode.toString());
if (analyzer == null) {
throw new MapperParsingException("Analyzer [" + fieldNode.toString() + "] not found for search_analyzer setting on root type [" + type + "]");
}
docBuilder.searchAnalyzer(analyzer);
} else if ("search_quote_analyzer".equals(fieldName)) {
iterator.remove();
NamedAnalyzer analyzer = analysisService.analyzer(fieldNode.toString());
if (analyzer == null) {
throw new MapperParsingException("Analyzer [" + fieldNode.toString() + "] not found for search_analyzer setting on root type [" + type + "]");
}
docBuilder.searchQuoteAnalyzer(analyzer);
} else if ("analyzer".equals(fieldName)) {
iterator.remove();
NamedAnalyzer analyzer = analysisService.analyzer(fieldNode.toString());
if (analyzer == null) {
throw new MapperParsingException("Analyzer [" + fieldNode.toString() + "] not found for analyzer setting on root type [" + type + "]");
@ -235,11 +245,25 @@ public class DocumentMapperParser extends AbstractIndexComponent {
} else {
Mapper.TypeParser typeParser = rootTypeParsers.get(fieldName);
if (typeParser != null) {
iterator.remove();
docBuilder.put(typeParser.parse(fieldName, (Map<String, Object>) fieldNode, parserContext));
}
}
}
ImmutableMap<String, Object> attributes = ImmutableMap.of();
if (mapping.containsKey("_meta")) {
attributes = ImmutableMap.copyOf((Map<String, Object>) mapping.remove("_meta"));
}
docBuilder.meta(attributes);
if (!mapping.isEmpty()) {
StringBuilder remainingFields = new StringBuilder();
for (String key : mapping.keySet()) {
remainingFields.append(" [").append(key).append(" : ").append(mapping.get(key).toString()).append("]");
}
throw new MapperParsingException("Root type mapping not empty after parsing! Remaining fields:" + remainingFields.toString());
}
if (!docBuilder.hasIndexAnalyzer()) {
docBuilder.indexAnalyzer(analysisService.defaultIndexAnalyzer());
}
@ -250,12 +274,6 @@ public class DocumentMapperParser extends AbstractIndexComponent {
docBuilder.searchAnalyzer(analysisService.defaultSearchQuoteAnalyzer());
}
ImmutableMap<String, Object> attributes = ImmutableMap.of();
if (mapping.containsKey("_meta")) {
attributes = ImmutableMap.copyOf((Map<String, Object>) mapping.get("_meta"));
}
docBuilder.meta(attributes);
DocumentMapper documentMapper = docBuilder.build(this);
// update the source with the generated one
documentMapper.refreshSource();
@ -279,7 +297,6 @@ public class DocumentMapperParser extends AbstractIndexComponent {
// if we don't have any keys throw an exception
throw new MapperParsingException("malformed mapping no root object found");
}
String rootName = root.keySet().iterator().next();
Tuple<String, Map<String, Object>> mapping;
if (type == null || type.equals(rootName)) {
@ -287,7 +304,6 @@ public class DocumentMapperParser extends AbstractIndexComponent {
} else {
mapping = new Tuple<>(type, root);
}
return mapping;
}
}

108
src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.index.mapper.object;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.XStringField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
@ -180,63 +179,80 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
public static class TypeParser implements Mapper.TypeParser {
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
Map<String, Object> objectNode = node;
ObjectMapper.Builder builder = createBuilder(name);
for (Map.Entry<String, Object> entry : node.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder);
parseObjectProperties(name, fieldName, fieldNode, builder);
}
parseNested(name, node, builder);
return builder;
}
protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, Object fieldNode, ParserContext parserContext, ObjectMapper.Builder builder) {
if (fieldName.equals("dynamic")) {
String value = fieldNode.toString();
if (value.equalsIgnoreCase("strict")) {
builder.dynamic(Dynamic.STRICT);
} else {
builder.dynamic(nodeBooleanValue(fieldNode) ? Dynamic.TRUE : Dynamic.FALSE);
}
return true;
} else if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
return true;
} else if (fieldName.equals("properties")) {
if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
// nothing to do here, empty (to support "properties: []" case)
} else if (!(fieldNode instanceof Map)) {
throw new ElasticsearchParseException("properties must be a map type");
} else {
parseProperties(builder, (Map<String, Object>) fieldNode, parserContext);
}
return true;
}
return false;
}
protected static void parseObjectProperties(String name, String fieldName, Object fieldNode, ObjectMapper.Builder builder) {
if (fieldName.equals("path")) {
builder.pathType(parsePathType(name, fieldNode.toString()));
} else if (fieldName.equals("include_in_all")) {
builder.includeInAll(nodeBooleanValue(fieldNode));
}
}
protected static void parseNested(String name, Map<String, Object> node, ObjectMapper.Builder builder) {
boolean nested = false;
boolean nestedIncludeInParent = false;
boolean nestedIncludeInRoot = false;
for (Map.Entry<String, Object> entry : objectNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("dynamic")) {
String value = fieldNode.toString();
if (value.equalsIgnoreCase("strict")) {
builder.dynamic(Dynamic.STRICT);
} else {
builder.dynamic(nodeBooleanValue(fieldNode) ? Dynamic.TRUE : Dynamic.FALSE);
}
} else if (fieldName.equals("type")) {
String type = fieldNode.toString();
if (type.equals(CONTENT_TYPE)) {
builder.nested = Nested.NO;
} else if (type.equals(NESTED_CONTENT_TYPE)) {
nested = true;
} else {
throw new MapperParsingException("Trying to parse an object but has a different type [" + type + "] for [" + name + "]");
}
} else if (fieldName.equals("include_in_parent")) {
nestedIncludeInParent = nodeBooleanValue(fieldNode);
} else if (fieldName.equals("include_in_root")) {
nestedIncludeInRoot = nodeBooleanValue(fieldNode);
} else if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("path")) {
builder.pathType(parsePathType(name, fieldNode.toString()));
} else if (fieldName.equals("properties")) {
if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
// nothing to do here, empty (to support "properties: []" case)
} else if (!(fieldNode instanceof Map)) {
throw new ElasticsearchParseException("properties must be a map type");
} else {
parseProperties(builder, (Map<String, Object>) fieldNode, parserContext);
}
} else if (fieldName.equals("include_in_all")) {
builder.includeInAll(nodeBooleanValue(fieldNode));
Object fieldNode = node.get("type");
if (fieldNode!=null) {
String type = fieldNode.toString();
if (type.equals(CONTENT_TYPE)) {
builder.nested = Nested.NO;
} else if (type.equals(NESTED_CONTENT_TYPE)) {
nested = true;
} else {
processField(builder, fieldName, fieldNode);
throw new MapperParsingException("Trying to parse an object but has a different type [" + type + "] for [" + name + "]");
}
}
fieldNode = node.get("include_in_parent");
if (fieldNode != null) {
nestedIncludeInParent = nodeBooleanValue(fieldNode);
}
fieldNode = node.get("include_in_root");
if (fieldNode != null) {
nestedIncludeInRoot = nodeBooleanValue(fieldNode);
}
if (nested) {
builder.nested = Nested.newNested(nestedIncludeInParent, nestedIncludeInRoot);
}
return builder;
}
private void parseProperties(ObjectMapper.Builder objBuilder, Map<String, Object> propsNode, ParserContext parserContext) {
protected static void parseProperties(ObjectMapper.Builder objBuilder, Map<String, Object> propsNode, ParserContext parserContext) {
for (Map.Entry<String, Object> entry : propsNode.entrySet()) {
String propName = entry.getKey();
Map<String, Object> propNode = (Map<String, Object>) entry.getValue();
@ -270,10 +286,6 @@ public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll {
protected Builder createBuilder(String name) {
return object(name);
}
protected void processField(Builder builder, String fieldName, Object fieldNode) {
}
}
private final String name;

29
src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.object;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.xcontent.ToXContent;
@ -29,10 +30,7 @@ import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.core.DateFieldMapper;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import static com.google.common.collect.Lists.newArrayList;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
@ -124,7 +122,23 @@ public class RootObjectMapper extends ObjectMapper {
}
@Override
protected void processField(ObjectMapper.Builder builder, String fieldName, Object fieldNode) {
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
ObjectMapper.Builder builder = createBuilder(name);
Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)
|| processField(builder, fieldName, fieldNode)) {
iterator.remove();
}
}
return builder;
}
protected boolean processField(ObjectMapper.Builder builder, String fieldName, Object fieldNode) {
if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) {
List<FormatDateTimeFormatter> dateTimeFormatters = newArrayList();
if (fieldNode instanceof List) {
@ -141,6 +155,7 @@ public class RootObjectMapper extends ObjectMapper {
} else {
((Builder) builder).dynamicDateTimeFormatter(dateTimeFormatters);
}
return true;
} else if (fieldName.equals("dynamic_templates")) {
// "dynamic_templates" : [
// {
@ -160,11 +175,15 @@ public class RootObjectMapper extends ObjectMapper {
Map.Entry<String, Object> entry = tmpl.entrySet().iterator().next();
((Builder) builder).add(DynamicTemplate.parse(entry.getKey(), (Map<String, Object>) entry.getValue()));
}
return true;
} else if (fieldName.equals("date_detection")) {
((Builder) builder).dateDetection = nodeBooleanValue(fieldNode);
return true;
} else if (fieldName.equals("numeric_detection")) {
((Builder) builder).numericDetection = nodeBooleanValue(fieldNode);
return true;
}
return false;
}
}

62
src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
View File

@ -22,7 +22,9 @@ package org.elasticsearch.index.mapper.all;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.lucene.all.AllEntries;
@ -33,18 +35,15 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperTestUtils;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.index.mapper.internal.*;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.hamcrest.Matchers;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.*;
import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath;
import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
@ -323,4 +322,55 @@ public class SimpleAllMapperTests extends ElasticsearchTestCase {
assertThat(allEntries.fields(), hasSize(1));
assertThat(allEntries.fields(), hasItem("foo.bar"));
}
@Test(expected = MapperParsingException.class)
public void testMisplacedTypeInRoot() throws IOException {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/misplaced_type_in_root.json");
DocumentMapper docMapper = MapperTestUtils.newParser().parse("test", mapping);
}
// related to https://github.com/elasticsearch/elasticsearch/issues/5864
@Test(expected = MapperParsingException.class)
public void testMistypedTypeInRoot() throws IOException {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mistyped_type_in_root.json");
DocumentMapper docMapper = MapperTestUtils.newParser().parse("test", mapping);
}
// issue https://github.com/elasticsearch/elasticsearch/issues/5864
@Test(expected = MapperParsingException.class)
public void testMisplacedMappingAsRoot() throws IOException {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/misplaced_mapping_key_in_root.json");
DocumentMapper docMapper = MapperTestUtils.newParser().parse("test", mapping);
}
// issue https://github.com/elasticsearch/elasticsearch/issues/5864
// test that RootObjectMapping still works
@Test
public void testRootObjectMapperPropertiesDoNotCauseException() throws IOException {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/type_dynamic_template_mapping.json");
MapperTestUtils.newParser().parse("test", mapping);
mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/type_dynamic_date_formats_mapping.json");
MapperTestUtils.newParser().parse("test", mapping);
mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/type_date_detection_mapping.json");
MapperTestUtils.newParser().parse("test", mapping);
mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/type_numeric_detection_mapping.json");
MapperTestUtils.newParser().parse("test", mapping);
}
// issue https://github.com/elasticsearch/elasticsearch/issues/5864
@Test
public void testRootMappersStillWorking() {
String mapping = "{";
Map<String, String> rootTypes = new HashMap<>();
//just pick some example from DocumentMapperParser.rootTypeParsers
rootTypes.put(SizeFieldMapper.NAME, "{\"enabled\" : true}");
rootTypes.put(IndexFieldMapper.NAME, "{\"enabled\" : true}");
rootTypes.put(SourceFieldMapper.NAME, "{\"enabled\" : true}");
rootTypes.put(TypeFieldMapper.NAME, "{\"store\" : true}");
for (String key : rootTypes.keySet()) {
mapping += "\"" + key+ "\"" + ":" + rootTypes.get(key) + ",\n";
}
mapping += "\"properties\":{}}" ;
MapperTestUtils.newParser().parse("test", mapping);
}
}

11
src/test/java/org/elasticsearch/index/mapper/all/misplaced_mapping_key_in_root.json Normal file
View File

@ -0,0 +1,11 @@
{
"mapping": {
"test": {
"properties": {
"foo": {
"type": "string"
}
}
}
}
}

8
src/test/java/org/elasticsearch/index/mapper/all/misplaced_type_in_root.json Normal file
View File

@ -0,0 +1,8 @@
{
"type": "string",
"properties": {
"foo": {
"type": "string"
}
}
}

9
src/test/java/org/elasticsearch/index/mapper/all/mistyped_type_in_root.json Normal file
View File

@ -0,0 +1,9 @@
{
"testX": {
"properties": {
"foo": {
"type": "string"
}
}
}
}

8
src/test/java/org/elasticsearch/index/mapper/all/type_date_detection_mapping.json Normal file
View File

@ -0,0 +1,8 @@
{
"date_detection" : false,
"properties": {
"foo": {
"type": "string"
}
}
}

8
src/test/java/org/elasticsearch/index/mapper/all/type_dynamic_date_formats_mapping.json Normal file
View File

@ -0,0 +1,8 @@
{
"dynamic_date_formats" : ["yyyy-MM-dd", "dd-MM-yyyy"],
"properties": {
"foo": {
"type": "string"
}
}
}

17
src/test/java/org/elasticsearch/index/mapper/all/type_dynamic_template_mapping.json Normal file
View File

@ -0,0 +1,17 @@
{
"dynamic_templates" : [
{
"dynamic_template_name" : {
"match" : "*",
"mapping" : {
"store" : true
}
}
}
],
"properties": {
"foo": {
"type": "string"
}
}
}

8
src/test/java/org/elasticsearch/index/mapper/all/type_numeric_detection_mapping.json Normal file
View File

@ -0,0 +1,8 @@
{
"numeric_detection" : false,
"properties": {
"foo": {
"type": "string"
}
}
}

2
src/test/java/org/elasticsearch/percolator/PercolatorTests.java
View File

@ -1686,7 +1686,7 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
.field("incude_in_all", false)
.endObject()
.endObject()
.startArray("dynamic_template")
.startArray("dynamic_templates")
.startObject()
.startObject("custom_fields")
.field("path_match", "custom.*")

4
src/test/java/org/elasticsearch/routing/SimpleRoutingTests.java
View File

@ -234,8 +234,8 @@ public class SimpleRoutingTests extends ElasticsearchIntegrationTest {
public void testRequiredRoutingWithPathMapping() throws Exception {
client().admin().indices().prepareCreate("test")
.addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("_routing").field("required", true).field("path", "routing_field").endObject()
.startObject("routing_field").field("type", "string").field("index", randomBoolean() ? "no" : "not_analyzed").field("doc_values", randomBoolean() ? "yes" : "no").endObject()
.startObject("_routing").field("required", true).field("path", "routing_field").endObject().startObject("properties")
.startObject("routing_field").field("type", "string").field("index", randomBoolean() ? "no" : "not_analyzed").field("doc_values", randomBoolean() ? "yes" : "no").endObject().endObject()
.endObject().endObject())
.execute().actionGet();
ensureGreen();

3
src/test/java/org/elasticsearch/search/sort/SimpleSortTests.java
View File

@ -848,9 +848,6 @@ public class SimpleSortTests extends ElasticsearchIntegrationTest {
.startObject("fielddata").field("format", maybeDocValues() ? "doc_values" : null).endObject()
.endObject()
.endObject()
.startObject("d_value")
.field("type", "float")
.endObject()
.endObject()
.endObject()));
ensureGreen();