Mapping: Revise dynamic mapping (into default), merge default to new mappings, closes #275.

.idea/dictionaries/kimchy.xml

@@ -95,6 +95,7 @@
       <w>queryparser</w>
       <w>rackspace</w>
       <w>rebalance</w>
+      <w>reparse</w>
       <w>retrans</w>
       <w>retval</w>
       <w>rsts</w>

modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java

@@ -147,7 +147,7 @@ public class MetaDataMappingService extends AbstractComponent {
                     } else if (!mappingType.equals(newMappers.values().iterator().next().type())) {
                         throw new InvalidTypeNameException("Type name provided does not match type name within mapping definition");
                     }
-                    if (mappingType.charAt(0) == '_') {
+                    if (!MapperService.DEFAULT_MAPPING.equals(mappingType) && mappingType.charAt(0) == '_') {
                         throw new InvalidTypeNameException("Document mapping type name can't start with '_'");
                     }
 

modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/XContentMerger.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import java.util.Map;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class XContentMerger {
+
+    /**
+     * Merges the defaults provided as the second parameter into the content of the first. Only does recursive merge
+     * for inner maps.
+     */
+    @SuppressWarnings({"unchecked"}) public static void mergeDefaults(Map<String, Object> content, Map<String, Object> defaults) {
+        for (Map.Entry<String, Object> defaultEntry : defaults.entrySet()) {
+            if (!content.containsKey(defaultEntry.getKey())) {
+                // copy it over, it does not exists in the content
+                content.put(defaultEntry.getKey(), defaultEntry.getValue());
+            } else {
+                // in the content and in the default, only merge compound ones (maps)
+                if (content.get(defaultEntry.getKey()) instanceof Map && defaultEntry.getValue() instanceof Map) {
+                    mergeDefaults((Map<String, Object>) content.get(defaultEntry.getKey()), (Map<String, Object>) defaultEntry.getValue());
+                }
+            }
+        }
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java

@@ -26,6 +26,11 @@ import javax.annotation.Nullable;
  */
 public interface DocumentMapperParser {
 
+    /**
+     * Parses the source mapping definition into a document mapper.
+     */
+    DocumentMapper parse(String mappingSource) throws MapperParsingException;
+
     /**
      * Parses the source mapping definition into a document mapper with the specified
      * type (overriding the one defined in the source mapping).
@@ -33,7 +38,8 @@ public interface DocumentMapperParser {
     DocumentMapper parse(@Nullable String type, String mappingSource) throws MapperParsingException;
 
     /**
-     * Parses the source mapping definition into a document mapper.
+     * Parses the source mapping definition into a document mapper with the specified
+     * type (overriding the one defined in the source mapping).
      */
-    DocumentMapper parse(String mappingSource) throws MapperParsingException;
+    DocumentMapper parse(@Nullable String type, String mappingSource, String defaultMappingSource) throws MapperParsingException;
 }

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -51,16 +51,14 @@ import static org.elasticsearch.common.collect.MapBuilder.*;
 @ThreadSafe
 public class MapperService extends AbstractIndexComponent implements Iterable<DocumentMapper> {
 
+    public static final String DEFAULT_MAPPING = "_default_";
+
     /**
      * Will create types automatically if they do not exists in the repo yet
      */
     private final boolean dynamic;
 
-    private final String dynamicMappingLocation;
+    private volatile String defaultMappingSource;
-
-    private final URL dynamicMappingUrl;
-
-    private final String dynamicMappingSource;
 
     private volatile ImmutableMap<String, DocumentMapper> mappers = ImmutableMap.of();
 
@@ -74,7 +72,7 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
     private volatile FieldMappers uidFieldMappers = new FieldMappers();
     private volatile FieldMappers sourceFieldMappers = new FieldMappers();
 
-    // for now, just use the json one. Can work on it more to support custom ones
+    // for now, just use the xcontent one. Can work on it more to support custom ones
     private final DocumentMapperParser documentParser;
 
     private final InternalFieldMapperListener fieldMapperListener = new InternalFieldMapperListener();
@@ -87,44 +85,35 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
         this.searchAnalyzer = new SmartIndexNameSearchAnalyzer(analysisService.defaultSearchAnalyzer());
 
         this.dynamic = componentSettings.getAsBoolean("dynamic", true);
-        String dynamicMappingLocation = componentSettings.get("dynamic_mapping_location");
+        String defaultMappingLocation = componentSettings.get("default_mapping_location");
-        URL dynamicMappingUrl;
+        URL defaultMappingUrl;
-        if (dynamicMappingLocation == null) {
+        if (defaultMappingLocation == null) {
             try {
-                dynamicMappingUrl = environment.resolveConfig("dynamic-mapping.json");
+                defaultMappingUrl = environment.resolveConfig("default-mapping.json");
             } catch (FailedToResolveConfigException e) {
                 // not there, default to the built in one
-                dynamicMappingUrl = indexSettings.getClassLoader().getResource("org/elasticsearch/index/mapper/xcontent/dynamic-mapping.json");
+                defaultMappingUrl = indexSettings.getClassLoader().getResource("org/elasticsearch/index/mapper/xcontent/default-mapping.json");
             }
         } else {
             try {
-                dynamicMappingUrl = environment.resolveConfig(dynamicMappingLocation);
+                defaultMappingUrl = environment.resolveConfig(defaultMappingLocation);
             } catch (FailedToResolveConfigException e) {
                 // not there, default to the built in one
                 try {
-                    dynamicMappingUrl = new File(dynamicMappingLocation).toURI().toURL();
+                    defaultMappingUrl = new File(defaultMappingLocation).toURI().toURL();
                 } catch (MalformedURLException e1) {
-                    throw new FailedToResolveConfigException("Failed to resolve dynamic mapping location [" + dynamicMappingLocation + "]");
+                    throw new FailedToResolveConfigException("Failed to resolve dynamic mapping location [" + defaultMappingLocation + "]");
                 }
             }
         }
-        this.dynamicMappingUrl = dynamicMappingUrl;
+
-        if (dynamicMappingLocation == null) {
+        try {
-            this.dynamicMappingLocation = dynamicMappingUrl.toExternalForm();
+            defaultMappingSource = Streams.copyToString(new InputStreamReader(defaultMappingUrl.openStream(), "UTF-8"));
-        } else {
+        } catch (IOException e) {
-            this.dynamicMappingLocation = dynamicMappingLocation;
+            throw new MapperException("Failed to load default mapping source from [" + defaultMappingLocation + "]", e);
         }
 
-        if (dynamic) {
+        logger.debug("using dynamic[{}], default mapping: location[{}] and source[{}]", dynamic, defaultMappingLocation, defaultMappingSource);
-            try {
-                dynamicMappingSource = Streams.copyToString(new InputStreamReader(dynamicMappingUrl.openStream(), "UTF-8"));
-            } catch (IOException e) {
-                throw new MapperException("Failed to load default mapping source from [" + dynamicMappingLocation + "]", e);
-            }
-        } else {
-            dynamicMappingSource = null;
-        }
-        logger.debug("using dynamic[{}] with location[{}] and source[{}]", dynamic, dynamicMappingLocation, dynamicMappingSource);
     }
 
     @Override public UnmodifiableIterator<DocumentMapper> iterator() {
@@ -145,7 +134,7 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
             if (mapper != null) {
                 return mapper;
             }
-            add(type, dynamicMappingSource);
+            add(type, null);
             return mappers.get(type);
         }
     }
@@ -155,18 +144,30 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
     }
 
     public void add(String type, String mappingSource) {
-        add(documentParser.parse(type, mappingSource));
+        if (DEFAULT_MAPPING.equals(type)) {
+            // verify we can parse it
+            documentParser.parse(type, mappingSource);
+            defaultMappingSource = mappingSource;
+        } else {
+            add(parse(type, mappingSource));
+        }
     }
 
-    public void add(String mappingSource) throws MapperParsingException {
+    private void add(DocumentMapper mapper) {
-        add(documentParser.parse(mappingSource));
+        synchronized (mutex) {
+            if (mapper.type().charAt(0) == '_') {
+                throw new InvalidTypeNameException("Document mapping type name can't start with '_'");
+            }
+            mappers = newMapBuilder(mappers).put(mapper.type(), mapper).immutableMap();
+            mapper.addFieldMapperListener(fieldMapperListener, true);
+        }
     }
 
     /**
      * Just parses and returns the mapper without adding it.
      */
     public DocumentMapper parse(String mappingType, String mappingSource) throws MapperParsingException {
-        return documentParser.parse(mappingType, mappingSource);
+        return documentParser.parse(mappingType, mappingSource, defaultMappingSource);
     }
 
     public boolean hasMapping(String mappingType) {
@@ -296,16 +297,6 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
         return null;
     }
 
-    public void add(DocumentMapper mapper) {
-        synchronized (mutex) {
-            if (mapper.type().charAt(0) == '_') {
-                throw new InvalidTypeNameException("Document mapping type name can't start with '_'");
-            }
-            mappers = newMapBuilder(mappers).put(mapper.type(), mapper).immutableMap();
-            mapper.addFieldMapperListener(fieldMapperListener, true);
-        }
-    }
-
     public Analyzer searchAnalyzer() {
         return this.searchAnalyzer;
     }

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapper.java

@@ -308,19 +308,19 @@ public class XContentDocumentMapper implements DocumentMapper, ToXContent {
             }
             token = parser.nextToken();
             if (token != XContentParser.Token.FIELD_NAME) {
-                throw new MapperException("Malformed content, after first object, the type name must exists");
+                throw new MapperException("Malformed content, after first object, either the type field or the actual properties should exist");
             }
-            if (!parser.currentName().equals(type)) {
+            if (parser.currentName().equals(type)) {
-                if (type == null) {
+                // first field is the same as the type, this might be because the type is provided, and the object exists within it
-                    throw new MapperException("Content _type [" + parser.currentName() + "] does not match the type of the mapper [" + type + "]");
+                // or because there is a valid field that by chance is named as the type
-                }
+
-                // continue
+                // Note, in this case, we only handle plain value types, an object type will be analyzed as if it was the type itself
-            } else {
+                // and other same level fields will be ignored
-                // now move to the actual content, which is the start object
                 token = parser.nextToken();
-                if (token != XContentParser.Token.START_OBJECT) {
+                // commented out, allow for same type with START_OBJECT, we do our best to handle it except for the above corner case
-                    throw new MapperException("Malformed content, a field with the same name as the type much be an object with the properties/fields within it");
+//                if (token != XContentParser.Token.START_OBJECT) {
-                }
+//                    throw new MapperException("Malformed content, a field with the same name as the type must be an object with the properties/fields within it");
+//                }
             }
 
             if (sourceFieldMapper.enabled()) {

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java

@@ -22,12 +22,15 @@ package org.elasticsearch.index.mapper.xcontent;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableMap;
 import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.collect.Maps;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentMerger;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.AnalysisService;
-import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
 
 import javax.annotation.Nullable;
 import java.io.IOException;
@@ -76,54 +79,41 @@ public class XContentDocumentMapperParser implements DocumentMapperParser {
         }
     }
 
-    @Override public DocumentMapper parse(String source) throws MapperParsingException {
+    @Override public XContentDocumentMapper parse(String source) throws MapperParsingException {
         return parse(null, source);
     }
 
-    @Override public DocumentMapper parse(@Nullable String type, String source) throws MapperParsingException {
+    @Override public XContentDocumentMapper parse(@Nullable String type, String source) throws MapperParsingException {
-        Map<String, Object> root;
+        return parse(type, source, null);
-        XContentParser xContentParser = null;
+    }
-        try {
+
-            xContentParser = XContentFactory.xContent(source).createParser(source);
+    @Override public XContentDocumentMapper parse(@Nullable String type, String source, String defaultSource) throws MapperParsingException {
-            root = xContentParser.map();
+        Map<String, Object> mapping = null;
-        } catch (IOException e) {
+        if (source != null) {
-            throw new MapperParsingException("Failed to parse mapping definition", e);
+            Tuple<String, Map<String, Object>> t = extractMapping(type, source);
-        } finally {
+            type = t.v1();
-            if (xContentParser != null) {
+            mapping = t.v2();
-                xContentParser.close();
-            }
         }
-        String rootName = root.keySet().iterator().next();
+        if (mapping == null) {
-        Map<String, Object> rootObj;
+            mapping = Maps.newHashMap();
+        }
+
         if (type == null) {
-            // we have no type, we assume the first node is the type
+            throw new MapperParsingException("Failed to derive type");
-            rootObj = (Map<String, Object>) root.get(rootName);
+        }
-            type = rootName;
+
-        } else {
+        if (defaultSource != null) {
-            // we have a type, check if the top level one is the type as well
+            Tuple<String, Map<String, Object>> t = extractMapping(MapperService.DEFAULT_MAPPING, defaultSource);
-            // if it is, then the root is that node, if not then the root is the master node
+            if (t.v2() != null) {
-            if (type.equals(rootName)) {
+                XContentMerger.mergeDefaults(mapping, t.v2());
-                Object tmpNode = root.get(type);
-                if (!(tmpNode instanceof Map)) {
-                    throw new MapperParsingException("Expected root node name [" + rootName + "] to be of object type, but its not");
-                }
-                rootObj = (Map<String, Object>) tmpNode;
-            } else if (rootName.equals("_default_")) {
-                Object tmpNode = root.get("_default_");
-                if (!(tmpNode instanceof Map)) {
-                    throw new MapperParsingException("_default_ mappings must have an inner object representing the actual mappings for the type");
-                }
-                rootObj = (Map<String, Object>) tmpNode;
-            } else {
-                rootObj = root;
             }
         }
 
-        XContentTypeParser.ParserContext parserContext = new XContentTypeParser.ParserContext(rootObj, analysisService, typeParsers);
+        XContentTypeParser.ParserContext parserContext = new XContentTypeParser.ParserContext(mapping, analysisService, typeParsers);
 
-        XContentDocumentMapper.Builder docBuilder = doc((XContentObjectMapper.Builder) rootObjectTypeParser.parse(type, rootObj, parserContext));
+        XContentDocumentMapper.Builder docBuilder = doc((XContentObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext));
 
-        for (Map.Entry<String, Object> entry : rootObj.entrySet()) {
+        for (Map.Entry<String, Object> entry : mapping.entrySet()) {
             String fieldName = Strings.toUnderscoreCase(entry.getKey());
             Object fieldNode = entry.getValue();
 
@@ -157,13 +147,11 @@ public class XContentDocumentMapperParser implements DocumentMapperParser {
         }
 
         ImmutableMap<String, Object> attributes = ImmutableMap.of();
-        if (rootObj.containsKey("_attributes")) {
+        if (mapping.containsKey("_attributes")) {
-            attributes = ImmutableMap.copyOf((Map<String, Object>) rootObj.get("_attributes"));
+            attributes = ImmutableMap.copyOf((Map<String, Object>) mapping.get("_attributes"));
         }
         docBuilder.attributes(attributes);
 
-        docBuilder.mappingSource(source);
-
         XContentDocumentMapper documentMapper = docBuilder.build();
         // update the source with the generated one
         documentMapper.mappingSource(documentMapper.buildSource());
@@ -227,4 +215,31 @@ public class XContentDocumentMapperParser implements DocumentMapperParser {
         }
         return builder;
     }
+
+    private Tuple<String, Map<String, Object>> extractMapping(String type, String source) throws MapperParsingException {
+        Map<String, Object> root;
+        XContentParser xContentParser = null;
+        try {
+            xContentParser = XContentFactory.xContent(source).createParser(source);
+            root = xContentParser.map();
+        } catch (IOException e) {
+            throw new MapperParsingException("Failed to parse mapping definition", e);
+        } finally {
+            if (xContentParser != null) {
+                xContentParser.close();
+            }
+        }
+
+        // we always assume the first and single key is the mapping type root
+        if (root.keySet().size() != 1) {
+            throw new MapperParsingException("Mapping must have the `type` as the root object");
+        }
+
+        String rootName = root.keySet().iterator().next();
+        if (type == null) {
+            type = rootName;
+        }
+
+        return new Tuple<String, Map<String, Object>>(type, (Map<String, Object>) root.get(rootName));
+    }
 }

modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentObjectMapper.java

@@ -314,8 +314,16 @@ public class XContentObjectMapper implements XContentMapper, XContentIncludeInAl
         context.path().pathType(pathType);
 
         String currentFieldName = parser.currentName();
-        XContentParser.Token token;
+        XContentParser.Token token = parser.currentToken();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+        // if we are at the end of the previous object, advance
+        if (token == XContentParser.Token.END_OBJECT) {
+            token = parser.nextToken();
+        }
+        if (token == XContentParser.Token.START_OBJECT) {
+            // if we are just starting an OBJECT, advance, this is the object we are parsing, we need the name first
+            token = parser.nextToken();
+        }
+        while (token != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.START_OBJECT) {
                 serializeObject(context, currentFieldName);
             } else if (token == XContentParser.Token.START_ARRAY) {
@@ -324,9 +332,10 @@ public class XContentObjectMapper implements XContentMapper, XContentIncludeInAl
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_NULL) {
                 serializeNullValue(context, currentFieldName);
-            } else {
+            } else if (token.isValue()) {
                 serializeValue(context, currentFieldName, token);
             }
+            token = parser.nextToken();
         }
         // restore the enable path flag
         context.path().pathType(origPathType);

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/XContentMapperTests.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.xcontent;
+
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.mapper.MapperService;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class XContentMapperTests {
+
+    public static XContentDocumentMapperParser newParser() {
+        return new XContentDocumentMapperParser(new AnalysisService(new Index("test")));
+    }
+
+    public static MapperService newMapperService() {
+        return new MapperService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS, new Environment(), new AnalysisService(new Index("test")));
+    }
+}

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/all/SimpleAllMapperTests.java

@@ -23,10 +23,8 @@ import org.apache.lucene.document.Document;
 import org.elasticsearch.common.lucene.all.AllEntries;
 import org.elasticsearch.common.lucene.all.AllField;
 import org.elasticsearch.common.lucene.all.AllTokenStream;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
-import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapperParser;
+import org.elasticsearch.index.mapper.xcontent.XContentMapperTests;
 import org.testng.annotations.Test;
 
 import static org.elasticsearch.common.io.Streams.*;
@@ -41,7 +39,7 @@ public class SimpleAllMapperTests {
 
     @Test public void testSimpleAllMappers() throws Exception {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/mapping.json");
-        XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
+        XContentDocumentMapper docMapper = XContentMapperTests.newParser().parse(mapping);
         byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
         Document doc = docMapper.parse(json).doc();
         AllField field = (AllField) doc.getFieldable("_all");
@@ -53,11 +51,11 @@ public class SimpleAllMapperTests {
 
     @Test public void testSimpleAllMappersWithReparse() throws Exception {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/mapping.json");
-        XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
+        XContentDocumentMapper docMapper = XContentMapperTests.newParser().parse(mapping);
         String builtMapping = docMapper.buildSource();
 //        System.out.println(builtMapping);
         // reparse it
-        XContentDocumentMapper builtDocMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(builtMapping);
+        XContentDocumentMapper builtDocMapper = XContentMapperTests.newParser().parse(builtMapping);
         byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
         Document doc = builtDocMapper.parse(json).doc();
 
@@ -70,7 +68,7 @@ public class SimpleAllMapperTests {
 
     @Test public void testSimpleAllMappersWithStore() throws Exception {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/store-mapping.json");
-        XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
+        XContentDocumentMapper docMapper = XContentMapperTests.newParser().parse(mapping);
         byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
         Document doc = docMapper.parse(json).doc();
         AllField field = (AllField) doc.getFieldable("_all");
@@ -85,11 +83,11 @@ public class SimpleAllMapperTests {
 
     @Test public void testSimpleAllMappersWithReparseWithStore() throws Exception {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/store-mapping.json");
-        XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
+        XContentDocumentMapper docMapper = XContentMapperTests.newParser().parse(mapping);
         String builtMapping = docMapper.buildSource();
         System.out.println(builtMapping);
         // reparse it
-        XContentDocumentMapper builtDocMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(builtMapping);
+        XContentDocumentMapper builtDocMapper = XContentMapperTests.newParser().parse(builtMapping);
         byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
         Document doc = builtDocMapper.parse(json).doc();
 

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/all/mapping.json

@@ -1,33 +1,33 @@
 {
-    person : {
+    "person" : {
-        _all : {enabled : true},
+        "_all" : {"enabled" : true},
-        properties : {
+        "properties" : {
-            name : {
+            "name" : {
-                type : "object",
+                "type" : "object",
-                dynamic : false,
+                "dynamic" : false,
-                properties : {
+                "properties" : {
-                    first : {type : "string", store : "yes", include_in_all : false},
+                    "first" : {"type" : "string", "store" : "yes", "include_in_all" : false},
-                    last : {type : "string", index : "not_analyzed"}
+                    "last" : {"type" : "string", "index" : "not_analyzed"}
                 }
             },
-            address : {
+            "address" : {
-                type : "object",
+                "type" : "object",
-                include_in_all : false,
+                "include_in_all" : false,
-                properties : {
+                "properties" : {
-                    first : {
+                    "first" : {
-                        properties : {
+                        "properties" : {
-                            location : {type : "string", store : "yes", index_name : "firstLocation"}
+                            "location" : {"type" : "string", "store" : "yes", "index_name" : "firstLocation"}
                         }
                     },
-                    last : {
+                    "last" : {
-                        properties : {
+                        "properties" : {
-                            location : {type : "string"}
+                            "location" : {"type" : "string"}
                         }
                     }
                 }
             },
-            simple1 : {type : "long", include_in_all : true},
+            "simple1" : {"type" : "long", "include_in_all" : true},
-            simple2 : {type : "long", include_in_all : false}
+            "simple2" : {"type" : "long", "include_in_all" : false}
         }
     }
 }

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/all/store-mapping.json

@@ -1,33 +1,33 @@
 {
-    person : {
+    "person" : {
-        _all : {enabled : true, store : "yes"},
+        "_all" : {"enabled" : true, "store" : "yes"},
-        properties : {
+        "properties" : {
-            name : {
+            "name" : {
-                type : "object",
+                "type" : "object",
-                dynamic : false,
+                "dynamic" : false,
-                properties : {
+                "properties" : {
-                    first : {type : "string", store : "yes", include_in_all : false},
+                    "first" : {"type" : "string", "store" : "yes", "include_in_all" : false},
-                    last : {type : "string", index : "not_analyzed"}
+                    "last" : {"type" : "string", "index" : "not_analyzed"}
                 }
             },
-            address : {
+            "address" : {
-                type : "object",
+                "type" : "object",
-                include_in_all : false,
+                "include_in_all" : false,
-                properties : {
+                "properties" : {
-                    first : {
+                    "first" : {
-                        properties : {
+                        "properties" : {
-                            location : {type : "string", store : "yes", index_name : "firstLocation"}
+                            "location" : {"type" : "string", "store" : "yes", "index_name" : "firstLocation"}
                         }
                     },
-                    last : {
+                    "last" : {
-                        properties : {
+                        "properties" : {
-                            location : {type : "string"}
+                            "location" : {"type" : "string"}
                         }
                     }
                 }
             },
-            simple1 : {type : "long", include_in_all : true},
+            "simple1" : {"type" : "long", "include_in_all" : true},
-            simple2 : {type : "long", include_in_all : false}
+            "simple2" : {"type" : "long", "include_in_all" : false}
         }
     }
 }

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/all/test1.json

@@ -1,20 +1,20 @@
 {
-    person : {
+    "person" : {
-        _boost : 3.7,
+        "_boost" : 3.7,
-        _id : "1",
+        "_id" : "1",
-        name : {
+        "name" : {
-            first : "shay",
+            "first" : "shay",
-            last : "banon"
+            "last" : "banon"
         },
-        address : {
+        "address" : {
-            first : {
+            "first" : {
-                location : "first location"
+                "location" : "first location"
             },
-            last : {
+            "last" : {
-                location : "last location"
+                "location" : "last location"
             }
         },
-        simple1 : 1,
+        "simple1" : 1,
-        simple2 : 2
+        "simple2" : 2
     }
 }

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/defaultsource/DefaultSourceMappingTests.java

@@ -0,0 +1,101 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.xcontent.defaultsource;
+
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
+import org.elasticsearch.index.mapper.xcontent.XContentMapperTests;
+import org.testng.annotations.Test;
+
+import static org.hamcrest.MatcherAssert.*;
+import static org.hamcrest.Matchers.*;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class DefaultSourceMappingTests {
+
+    @Test public void testDefaultMappingAndNoMapping() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject(MapperService.DEFAULT_MAPPING)
+                .startObject("_source").field("enabled", false).endObject()
+                .endObject().endObject().string();
+
+        XContentDocumentMapper mapper = XContentMapperTests.newParser().parse("my_type", null, defaultMapping);
+        assertThat(mapper.type(), equalTo("my_type"));
+        assertThat(mapper.sourceMapper().enabled(), equalTo(false));
+
+        try {
+            mapper = XContentMapperTests.newParser().parse(null, null, defaultMapping);
+            assertThat(mapper.type(), equalTo("my_type"));
+            assertThat(mapper.sourceMapper().enabled(), equalTo(false));
+            assert false;
+        } catch (MapperParsingException e) {
+            // all is well
+        }
+    }
+
+    @Test public void testDefaultMappingAndWithMappingOverride() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject(MapperService.DEFAULT_MAPPING)
+                .startObject("_source").field("enabled", false).endObject()
+                .endObject().endObject().string();
+
+        String mapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type")
+                .startObject("_source").field("enabled", true).endObject()
+                .endObject().endObject().string();
+
+        XContentDocumentMapper mapper = XContentMapperTests.newParser().parse("my_type", mapping, defaultMapping);
+        assertThat(mapper.type(), equalTo("my_type"));
+        assertThat(mapper.sourceMapper().enabled(), equalTo(true));
+    }
+
+    @Test public void testDefaultMappingAndNoMappingWithMapperService() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject(MapperService.DEFAULT_MAPPING)
+                .startObject("_source").field("enabled", false).endObject()
+                .endObject().endObject().string();
+
+        MapperService mapperService = XContentMapperTests.newMapperService();
+        mapperService.add(MapperService.DEFAULT_MAPPING, defaultMapping);
+
+        XContentDocumentMapper mapper = (XContentDocumentMapper) mapperService.type("my_type");
+        assertThat(mapper.type(), equalTo("my_type"));
+        assertThat(mapper.sourceMapper().enabled(), equalTo(false));
+    }
+
+    @Test public void testDefaultMappingAndWithMappingOverrideWithMapperService() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject(MapperService.DEFAULT_MAPPING)
+                .startObject("_source").field("enabled", false).endObject()
+                .endObject().endObject().string();
+
+        MapperService mapperService = XContentMapperTests.newMapperService();
+        mapperService.add(MapperService.DEFAULT_MAPPING, defaultMapping);
+
+        String mapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type")
+                .startObject("_source").field("enabled", true).endObject()
+                .endObject().endObject().string();
+        mapperService.add("my_type", mapping);
+
+        XContentDocumentMapper mapper = (XContentDocumentMapper) mapperService.type("my_type");
+        assertThat(mapper.type(), equalTo("my_type"));
+        assertThat(mapper.sourceMapper().enabled(), equalTo(true));
+    }
+}

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/merge/test1/Test1MergeMapperTests.java

@@ -19,11 +19,9 @@
 
 package org.elasticsearch.index.mapper.xcontent.merge.test1;
 
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
-import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapperParser;
+import org.elasticsearch.index.mapper.xcontent.XContentMapperTests;
 import org.testng.annotations.Test;
 
 import static org.elasticsearch.common.io.Streams.*;
@@ -39,9 +37,9 @@ public class Test1MergeMapperTests {
 
     @Test public void test1Merge() throws Exception {
         String stage1Mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/merge/test1/stage1.json");
-        XContentDocumentMapper stage1 = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(stage1Mapping);
+        XContentDocumentMapper stage1 = XContentMapperTests.newParser().parse(stage1Mapping);
         String stage2Mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/merge/test1/stage2.json");
-        XContentDocumentMapper stage2 = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(stage2Mapping);
+        XContentDocumentMapper stage2 = XContentMapperTests.newParser().parse(stage2Mapping);
 
         DocumentMapper.MergeResult mergeResult = stage1.merge(stage2, mergeFlags().simulate(true));
         assertThat(mergeResult.hasConflicts(), equalTo(false));

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/multifield/XContentMultiFieldTests.java

@@ -21,10 +21,8 @@ package org.elasticsearch.index.mapper.xcontent.multifield;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
-import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapperParser;
+import org.elasticsearch.index.mapper.xcontent.XContentMapperTests;
 import org.testng.annotations.Test;
 
 import static org.elasticsearch.common.io.Streams.*;
@@ -40,7 +38,7 @@ public class XContentMultiFieldTests {
 
     @Test public void testMultiField() throws Exception {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/multifield/test-mapping.json");
-        XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
+        XContentDocumentMapper docMapper = XContentMapperTests.newParser().parse(mapping);
         byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/multifield/test-data.json");
         Document doc = docMapper.parse(json).doc();
 
@@ -81,7 +79,7 @@ public class XContentMultiFieldTests {
         String builtMapping = builderDocMapper.buildSource();
 //        System.out.println(builtMapping);
         // reparse it
-        XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(builtMapping);
+        XContentDocumentMapper docMapper = XContentMapperTests.newParser().parse(builtMapping);
 
 
         byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/multifield/test-data.json");

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/multifield/test-data.json

@@ -1,7 +1,7 @@
 {
-    _id : 1,
+    "_id" : 1,
-    name : "some name",
+    "name" : "some name",
-    object1 : {
+    "object1" : {
-        multi1 : "2010-01-01"
+        "multi1" : "2010-01-01"
     }
 }

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/multifield/test-mapping.json

@@ -1,21 +1,21 @@
 {
-    person : {
+    "person" : {
-        properties : {
+        "properties" : {
             "name" : {
-                type : "multi_field",
+                "type" : "multi_field",
                 "fields" : {
-                    "name" : {type: "string", index : "analyzed", store : "yes"},
+                    "name" : {"type": "string", "index" : "analyzed", "store" : "yes"},
-                    "indexed" : {type: "string", index : "analyzed"},
+                    "indexed" : {"type": "string", "index" : "analyzed"},
-                    "not_indexed" : {type: "string", index : "no", store : "yes"}
+                    "not_indexed" : {"type": "string", "index" : "no", "store" : "yes"}
                 }
             },
             "object1" : {
-                properties : {
+                "properties" : {
                     "multi1" : {
-                        type : "multi_field",
+                        "type" : "multi_field",
                         "fields" : {
-                            "multi1" : {type : "date"},
+                            "multi1" : {"type": "date"},
-                            "string" : {type: "string", index : "not_analyzed"}
+                            "string" : {"type": "string", "index" : "not_analyzed"}
                         }
                     }
                 }

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/overridetype/OverrideTypeMappingTests.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.xcontent.overridetype;
+
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
+import org.elasticsearch.index.mapper.xcontent.XContentMapperTests;
+import org.testng.annotations.Test;
+
+import static org.hamcrest.MatcherAssert.*;
+import static org.hamcrest.Matchers.*;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class OverrideTypeMappingTests {
+
+    @Test public void testOverrideType() throws Exception {
+        String mapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type")
+                .startObject("_source").field("enabled", false).endObject()
+                .endObject().endObject().string();
+
+        XContentDocumentMapper mapper = XContentMapperTests.newParser().parse("my_type", mapping);
+        assertThat(mapper.type(), equalTo("my_type"));
+        assertThat(mapper.sourceMapper().enabled(), equalTo(false));
+
+        mapper = XContentMapperTests.newParser().parse(mapping);
+        assertThat(mapper.type(), equalTo("type"));
+        assertThat(mapper.sourceMapper().enabled(), equalTo(false));
+    }
+}

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/path/PathXContentMapperTests.java

@@ -19,10 +19,8 @@
 
 package org.elasticsearch.index.mapper.xcontent.path;
 
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
-import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapperParser;
+import org.elasticsearch.index.mapper.xcontent.XContentMapperTests;
 import org.testng.annotations.Test;
 
 import java.io.IOException;
@@ -38,7 +36,7 @@ public class PathXContentMapperTests {
 
     @Test public void testPathMapping() throws IOException {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/path/test-mapping.json");
-        XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
+        XContentDocumentMapper docMapper = XContentMapperTests.newParser().parse(mapping);
 
         assertThat(docMapper.mappers().indexName("first1"), notNullValue());
         assertThat(docMapper.mappers().indexName("name1.first1"), nullValue());

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/simple/SimpleXContentMapperTests.java

@@ -91,19 +91,6 @@ public class SimpleXContentMapperTests {
 //        System.out.println("Json: " + docMapper.sourceMapper().value(doc));
     }
 
-    @Test public void testSimpleParserMappingWithNoType() throws Exception {
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/simple/test-mapping-notype.json");
-        XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse("person", mapping);
-        byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/simple/test1.json");
-        Document doc = docMapper.parse(json).doc();
-        assertThat(doc.get(docMapper.uidMapper().names().indexName()), equalTo(Uid.createUid("person", "1")));
-        assertThat((double) doc.getBoost(), closeTo(3.7, 0.01));
-        assertThat(doc.get(docMapper.mappers().name("first").mapper().names().indexName()), equalTo("shay"));
-        assertThat(doc.getFields(docMapper.idMapper().names().indexName()).length, equalTo(1));
-//        System.out.println("Document: " + doc);
-//        System.out.println("Json: " + docMapper.sourceMapper().value(doc));
-    }
-
     @Test public void testSimpleParserNoTypeNoId() throws Exception {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/simple/test-mapping.json");
         XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/simple/test-mapping-notype.json

@@ -1,52 +0,0 @@
-{
-    date_formats : ["yyyy-MM-dd", "dd-MM-yyyy"],
-    dynamic : false,
-    enabled : true,
-    _id : {name : "_id", index_name : "_id"},
-    _source : {name : "_source" },
-    _type : {name : "_type"},
-    _boost : {name : "_boost", null_value : 2.0},
-    properties : {
-        name : {
-            type : "object",
-            dynamic : false,
-            properties : {
-                first : {type : "string", store : "yes"},
-                last : {type : "string", index : "not_analyzed"}
-            }
-        },
-        address : {
-            type : "object",
-            properties : {
-                first : {
-                    properties : {
-                        location : {type : "string", store : "yes", index_name : "firstLocation"}
-                    }
-                },
-                last : {
-                    properties : {
-                        location : {type : "string"}
-                    }
-                }
-            }
-        },
-        age : {type : "integer", null_value : 0},
-        birthdate : {type : "date", format : "yyyy-MM-dd"},
-        nerd : {type : "boolean"},
-        dogs : {type : "string", index_name : "dog"},
-        complex : {
-            type : "object",
-            properties : {
-                value1 : {type : "string"},
-                value2 : {type : "string"}
-            }
-        },
-        complex2 : {
-            type : "object",
-            properties : {
-                value1 : {type : "string"},
-                value2 : {type : "string"}
-            }
-        }
-    }
-}

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/typelevels/ParseDocumentTypeLevelsTests.java

@@ -0,0 +1,233 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.xcontent.typelevels;
+
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
+import org.elasticsearch.index.mapper.xcontent.XContentMapperTests;
+import org.testng.annotations.Test;
+
+import static org.hamcrest.MatcherAssert.*;
+import static org.hamcrest.Matchers.*;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class ParseDocumentTypeLevelsTests {
+
+    @Test public void testNoLevel() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type").endObject().endObject().string();
+
+        XContentDocumentMapper defaultMapper = XContentMapperTests.newParser().parse(defaultMapping);
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("test1", "value1")
+                .field("test2", "value2")
+                .startObject("inner").field("inner_field", "inner_value").endObject()
+                .endObject()
+                .copiedBytes());
+
+        assertThat(doc.doc().get("test1"), equalTo("value1"));
+        assertThat(doc.doc().get("test2"), equalTo("value2"));
+        assertThat(doc.doc().get("inner.inner_field"), equalTo("inner_value"));
+    }
+
+    @Test public void testTypeLevel() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type").endObject().endObject().string();
+
+        XContentDocumentMapper defaultMapper = XContentMapperTests.newParser().parse(defaultMapping);
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject().startObject("type")
+                .field("test1", "value1")
+                .field("test2", "value2")
+                .startObject("inner").field("inner_field", "inner_value").endObject()
+                .endObject().endObject()
+                .copiedBytes());
+
+        assertThat(doc.doc().get("test1"), equalTo("value1"));
+        assertThat(doc.doc().get("test2"), equalTo("value2"));
+        assertThat(doc.doc().get("inner.inner_field"), equalTo("inner_value"));
+    }
+
+    @Test public void testNoLevelWithFieldTypeAsValue() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type").endObject().endObject().string();
+
+        XContentDocumentMapper defaultMapper = XContentMapperTests.newParser().parse(defaultMapping);
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("type", "value_type")
+                .field("test1", "value1")
+                .field("test2", "value2")
+                .startObject("inner").field("inner_field", "inner_value").endObject()
+                .endObject()
+                .copiedBytes());
+
+        assertThat(doc.doc().get("type"), equalTo("value_type"));
+        assertThat(doc.doc().get("test1"), equalTo("value1"));
+        assertThat(doc.doc().get("test2"), equalTo("value2"));
+        assertThat(doc.doc().get("inner.inner_field"), equalTo("inner_value"));
+    }
+
+    @Test public void testTypeLevelWithFieldTypeAsValue() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type").endObject().endObject().string();
+
+        XContentDocumentMapper defaultMapper = XContentMapperTests.newParser().parse(defaultMapping);
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject().startObject("type")
+                .field("type", "value_type")
+                .field("test1", "value1")
+                .field("test2", "value2")
+                .startObject("inner").field("inner_field", "inner_value").endObject()
+                .endObject().endObject()
+                .copiedBytes());
+
+        assertThat(doc.doc().get("type"), equalTo("value_type"));
+        assertThat(doc.doc().get("test1"), equalTo("value1"));
+        assertThat(doc.doc().get("test2"), equalTo("value2"));
+        assertThat(doc.doc().get("inner.inner_field"), equalTo("inner_value"));
+    }
+
+    @Test public void testNoLevelWithFieldTypeAsObject() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type").endObject().endObject().string();
+
+        XContentDocumentMapper defaultMapper = XContentMapperTests.newParser().parse(defaultMapping);
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .startObject("type").field("type_field", "type_value").endObject()
+                .field("test1", "value1")
+                .field("test2", "value2")
+                .startObject("inner").field("inner_field", "inner_value").endObject()
+                .endObject()
+                .copiedBytes());
+
+        // in this case, we analyze the type object as the actual document, and ignore the other same level fields
+        assertThat(doc.doc().get("type_field"), equalTo("type_value"));
+        assertThat(doc.doc().get("test1"), nullValue());
+        assertThat(doc.doc().get("test2"), nullValue());
+    }
+
+    @Test public void testTypeLevelWithFieldTypeAsObject() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type").endObject().endObject().string();
+
+        XContentDocumentMapper defaultMapper = XContentMapperTests.newParser().parse(defaultMapping);
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject().startObject("type")
+                .startObject("type").field("type_field", "type_value").endObject()
+                .field("test1", "value1")
+                .field("test2", "value2")
+                .startObject("inner").field("inner_field", "inner_value").endObject()
+                .endObject().endObject()
+                .copiedBytes());
+
+        assertThat(doc.doc().get("type.type_field"), equalTo("type_value"));
+        assertThat(doc.doc().get("test1"), equalTo("value1"));
+        assertThat(doc.doc().get("test2"), equalTo("value2"));
+        assertThat(doc.doc().get("inner.inner_field"), equalTo("inner_value"));
+    }
+
+    @Test public void testNoLevelWithFieldTypeAsValueNotFirst() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type").endObject().endObject().string();
+
+        XContentDocumentMapper defaultMapper = XContentMapperTests.newParser().parse(defaultMapping);
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject().startObject("type")
+                .field("test1", "value1")
+                .field("test2", "value2")
+                .field("type", "value_type")
+                .startObject("inner").field("inner_field", "inner_value").endObject()
+                .endObject().endObject()
+                .copiedBytes());
+
+        assertThat(doc.doc().get("type"), equalTo("value_type"));
+        assertThat(doc.doc().get("test1"), equalTo("value1"));
+        assertThat(doc.doc().get("test2"), equalTo("value2"));
+        assertThat(doc.doc().get("inner.inner_field"), equalTo("inner_value"));
+    }
+
+    @Test public void testTypeLevelWithFieldTypeAsValueNotFirst() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type").endObject().endObject().string();
+
+        XContentDocumentMapper defaultMapper = XContentMapperTests.newParser().parse(defaultMapping);
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject().startObject("type")
+                .field("test1", "value1")
+                .field("type", "value_type")
+                .field("test2", "value2")
+                .startObject("inner").field("inner_field", "inner_value").endObject()
+                .endObject().endObject()
+                .copiedBytes());
+
+        assertThat(doc.doc().get("type"), equalTo("value_type"));
+        assertThat(doc.doc().get("test1"), equalTo("value1"));
+        assertThat(doc.doc().get("test2"), equalTo("value2"));
+        assertThat(doc.doc().get("inner.inner_field"), equalTo("inner_value"));
+    }
+
+    @Test public void testNoLevelWithFieldTypeAsObjectNotFirst() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type").endObject().endObject().string();
+
+        XContentDocumentMapper defaultMapper = XContentMapperTests.newParser().parse(defaultMapping);
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("test1", "value1")
+                .startObject("type").field("type_field", "type_value").endObject()
+                .field("test2", "value2")
+                .startObject("inner").field("inner_field", "inner_value").endObject()
+                .endObject()
+                .copiedBytes());
+
+        // when the type is not the first one, we don't confuse it...
+        assertThat(doc.doc().get("type.type_field"), equalTo("type_value"));
+        assertThat(doc.doc().get("test1"), equalTo("value1"));
+        assertThat(doc.doc().get("test2"), equalTo("value2"));
+        assertThat(doc.doc().get("inner.inner_field"), equalTo("inner_value"));
+    }
+
+    @Test public void testTypeLevelWithFieldTypeAsObjectNotFirst() throws Exception {
+        String defaultMapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type").endObject().endObject().string();
+
+        XContentDocumentMapper defaultMapper = XContentMapperTests.newParser().parse(defaultMapping);
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject().startObject("type")
+                .field("test1", "value1")
+                .startObject("type").field("type_field", "type_value").endObject()
+                .field("test2", "value2")
+                .startObject("inner").field("inner_field", "inner_value").endObject()
+                .endObject().endObject()
+                .copiedBytes());
+
+        assertThat(doc.doc().get("type.type_field"), equalTo("type_value"));
+        assertThat(doc.doc().get("test1"), equalTo("value1"));
+        assertThat(doc.doc().get("test2"), equalTo("value2"));
+        assertThat(doc.doc().get("inner.inner_field"), equalTo("inner_value"));
+    }
+}

modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/typelevels/ParseMappingTypeLevelTests.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.xcontent.typelevels;
+
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
+import org.elasticsearch.index.mapper.xcontent.XContentMapperTests;
+import org.testng.annotations.Test;
+
+import static org.hamcrest.MatcherAssert.*;
+import static org.hamcrest.Matchers.*;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class ParseMappingTypeLevelTests {
+
+    @Test public void testTypeLevel() throws Exception {
+        String mapping = XContentFactory.contentTextBuilder(XContentType.JSON).startObject().startObject("type")
+                .startObject("_source").field("enabled", false).endObject()
+                .endObject().endObject().string();
+
+        XContentDocumentMapper mapper = XContentMapperTests.newParser().parse("type", mapping);
+        assertThat(mapper.type(), equalTo("type"));
+        assertThat(mapper.sourceMapper().enabled(), equalTo(false));
+
+        mapper = XContentMapperTests.newParser().parse(mapping);
+        assertThat(mapper.type(), equalTo("type"));
+        assertThat(mapper.sourceMapper().enabled(), equalTo(false));
+    }
+}