Merge pull request #18220 from jimferenczi/_type_doc_values

Make doc_values accessible for _type

core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java

@@ -35,12 +35,16 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.StringFieldType;
+import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
@@ -87,12 +91,29 @@ public class TypeFieldMapper extends MetadataFieldMapper {
     }
 
     static final class TypeFieldType extends StringFieldType {
+        private boolean fielddata;
 
         public TypeFieldType() {
+            this.fielddata = false;
         }
 
         protected TypeFieldType(TypeFieldType ref) {
             super(ref);
+            this.fielddata = ref.fielddata;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (super.equals(o) == false) {
+                return false;
+            }
+            TypeFieldType that = (TypeFieldType) o;
+            return fielddata == that.fielddata;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(super.hashCode(), fielddata);
         }
 
         @Override
@@ -105,6 +126,29 @@ public class TypeFieldMapper extends MetadataFieldMapper {
             return CONTENT_TYPE;
         }
 
+        public boolean fielddata() {
+            return fielddata;
+        }
+
+        public void setFielddata(boolean fielddata) {
+            checkIfFrozen();
+            this.fielddata = fielddata;
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            if (hasDocValues()) {
+                return new DocValuesIndexFieldData.Builder();
+            }
+            assert indexOptions() != IndexOptions.NONE;
+            if (fielddata) {
+                return new PagedBytesIndexFieldData.Builder(TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
+                    TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
+                    TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
+            }
+            return super.fielddataBuilder();
+        }
+
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
             if (indexOptions() == IndexOptions.NONE) {
@@ -112,6 +156,19 @@ public class TypeFieldMapper extends MetadataFieldMapper {
             }
             return new TypeQuery(indexedValueForSearch(value));
         }
+
+        @Override
+        public void checkCompatibility(MappedFieldType other,
+                                       List<String> conflicts, boolean strict) {
+            super.checkCompatibility(other, conflicts, strict);
+            TypeFieldType otherType = (TypeFieldType) other;
+            if (strict) {
+                if (fielddata() != otherType.fielddata()) {
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [fielddata] "
+                        + "across all types.");
+                }
+            }
+        }
     }
 
     public static class TypeQuery extends Query {
@@ -169,7 +226,10 @@ public class TypeFieldMapper extends MetadataFieldMapper {
     private static MappedFieldType defaultFieldType(Settings indexSettings) {
         MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone();
         Version indexCreated = Version.indexCreated(indexSettings);
-        if (indexCreated.onOrAfter(Version.V_2_1_0)) {
+        if (indexCreated.before(Version.V_2_1_0)) {
+            // enables fielddata loading, doc values was disabled on _type between 2.0 and 2.1.
+            ((TypeFieldType) defaultFieldType).setFielddata(true);
+        } else {
             defaultFieldType.setHasDocValues(true);
         }
         return defaultFieldType;

core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java

@@ -24,6 +24,8 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -31,6 +33,8 @@ import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.util.Collection;
 
+import static org.hamcrest.Matchers.instanceOf;
+
 public class TypeFieldMapperTests extends ESSingleNodeTestCase {
 
     @Override
@@ -44,6 +48,7 @@ public class TypeFieldMapperTests extends ESSingleNodeTestCase {
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class);
         assertTrue(typeMapper.fieldType().hasDocValues());
+        assertThat(typeMapper.fieldType().fielddataBuilder(), instanceOf(DocValuesIndexFieldData.Builder.class));
     }
 
     public void testDocValuesPre21() throws Exception {
@@ -54,5 +59,6 @@ public class TypeFieldMapperTests extends ESSingleNodeTestCase {
         DocumentMapper docMapper = createIndex("test", bwcSettings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class);
         assertFalse(typeMapper.fieldType().hasDocValues());
+        assertThat(typeMapper.fieldType().fielddataBuilder(), instanceOf(PagedBytesIndexFieldData.Builder.class));
     }
 }

core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java

@@ -37,6 +37,7 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.junit.Before;
 
 public class TypeFieldTypeTests extends FieldTypeTestCase {
     @Override
@@ -44,6 +45,17 @@ public class TypeFieldTypeTests extends FieldTypeTestCase {
         return new TypeFieldMapper.TypeFieldType();
     }
 
+    @Before
+    public void setupProperties() {
+        addModifier(new Modifier("fielddata", true) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                TypeFieldMapper.TypeFieldType tft = (TypeFieldMapper.TypeFieldType) ft;
+                tft.setFielddata(tft.fielddata() == false);
+            }
+        });
+    }
+
     public void testTermQuery() throws Exception {
         Directory dir = newDirectory();
         IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());

docs/reference/mapping/fields/type-field.asciidoc

@@ -5,7 +5,8 @@ Each document indexed is associated with a <<mapping-type-field,`_type`>> (see
 <<mapping-type>>) and an <<mapping-id-field,`_id`>>.  The `_type` field is
 indexed in order to make searching by type name fast.
 
-The value of the `_type` field is accessible in queries and scripts:
+The value of the `_type` field is accessible in queries, aggregations,
+scripts, and when sorting:
 
 [source,js]
 --------------------------
@@ -27,9 +28,24 @@ GET my_index/type_*/_search
       "_type": [ "type_1", "type_2" ] <1>
     }
   },
+  "aggs": {
+    "types": {
+      "terms": {
+        "field": "_type", <2>
+        "size": 10
+      }
+    }
+  },
+  "sort": [
+    {
+      "_type": { <3>
+        "order": "desc"
+      }
+    }
+  ],
   "script_fields": {
     "type": {
-      "script": "doc['_type']" <2>
+      "script": "doc['_type']" <4>
     }
   }
 }
@@ -38,4 +54,7 @@ GET my_index/type_*/_search
 // CONSOLE
 
 <1> Querying on the `_type` field
-<2> Accessing the `_type` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+<2> Aggregating on the `_type` field
+<3> Sorting on the `_type` field
+<4> Accessing the `_type` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+