From 6d62f337021a6d47bbdc7940031a0e7f1e742106 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Mon, 9 May 2016 20:23:42 +0200 Subject: [PATCH] Make doc_values accessible for _type `doc_values` for _type field are created but any attempt to load them throws an IAE. This PR re-enables `doc_values` loading for _type, it also enables `fielddata` loading for indices created between 2.0 and 2.1 since doc_values were disabled during that period. It also restores the old docs that gives example on how to sort or aggregate on _type field. --- .../mapper/internal/TypeFieldMapper.java | 62 ++++++++++++++++++- .../mapper/internal/TypeFieldMapperTests.java | 6 ++ .../mapper/internal/TypeFieldTypeTests.java | 12 ++++ .../mapping/fields/type-field.asciidoc | 25 +++++++- 4 files changed, 101 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java index 7aee0aefad8..f960ecaa977 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java @@ -35,12 +35,16 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; +import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; +import org.elasticsearch.index.mapper.core.TextFieldMapper; import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; @@ -87,12 +91,29 @@ public class TypeFieldMapper extends MetadataFieldMapper { } static final class TypeFieldType extends StringFieldType { + private boolean fielddata; public TypeFieldType() { + this.fielddata = false; } protected TypeFieldType(TypeFieldType ref) { super(ref); + this.fielddata = ref.fielddata; + } + + @Override + public boolean equals(Object o) { + if (super.equals(o) == false) { + return false; + } + TypeFieldType that = (TypeFieldType) o; + return fielddata == that.fielddata; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), fielddata); } @Override @@ -105,6 +126,29 @@ public class TypeFieldMapper extends MetadataFieldMapper { return CONTENT_TYPE; } + public boolean fielddata() { + return fielddata; + } + + public void setFielddata(boolean fielddata) { + checkIfFrozen(); + this.fielddata = fielddata; + } + + @Override + public IndexFieldData.Builder fielddataBuilder() { + if (hasDocValues()) { + return new DocValuesIndexFieldData.Builder(); + } + assert indexOptions() != IndexOptions.NONE; + if (fielddata) { + return new PagedBytesIndexFieldData.Builder(TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, + TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY, + TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE); + } + return super.fielddataBuilder(); + } + @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { if (indexOptions() == IndexOptions.NONE) { @@ -112,6 +156,19 @@ public class TypeFieldMapper extends MetadataFieldMapper { } return new TypeQuery(indexedValueForSearch(value)); } + + @Override + public void checkCompatibility(MappedFieldType other, + List conflicts, boolean strict) { + super.checkCompatibility(other, conflicts, strict); + TypeFieldType otherType = (TypeFieldType) other; + if (strict) { + if (fielddata() != otherType.fielddata()) { + conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [fielddata] " + + "across all types."); + } + } + } } public static class TypeQuery extends Query { @@ -169,7 +226,10 @@ public class TypeFieldMapper extends MetadataFieldMapper { private static MappedFieldType defaultFieldType(Settings indexSettings) { MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone(); Version indexCreated = Version.indexCreated(indexSettings); - if (indexCreated.onOrAfter(Version.V_2_1_0)) { + if (indexCreated.before(Version.V_2_1_0)) { + // enables fielddata loading, doc values was disabled on _type between 2.0 and 2.1. + ((TypeFieldType) defaultFieldType).setFielddata(true); + } else { defaultFieldType.setHasDocValues(true); } return defaultFieldType; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java index 77e7b46537e..bd3aaf84a4f 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java @@ -24,6 +24,8 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; +import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -31,6 +33,8 @@ import org.elasticsearch.test.InternalSettingsPlugin; import java.util.Collection; +import static org.hamcrest.Matchers.instanceOf; + public class TypeFieldMapperTests extends ESSingleNodeTestCase { @Override @@ -44,6 +48,7 @@ public class TypeFieldMapperTests extends ESSingleNodeTestCase { DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class); assertTrue(typeMapper.fieldType().hasDocValues()); + assertThat(typeMapper.fieldType().fielddataBuilder(), instanceOf(DocValuesIndexFieldData.Builder.class)); } public void testDocValuesPre21() throws Exception { @@ -54,5 +59,6 @@ public class TypeFieldMapperTests extends ESSingleNodeTestCase { DocumentMapper docMapper = createIndex("test", bwcSettings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class); assertFalse(typeMapper.fieldType().hasDocValues()); + assertThat(typeMapper.fieldType().fielddataBuilder(), instanceOf(PagedBytesIndexFieldData.Builder.class)); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java index 91216983b70..8f87ca9d683 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java @@ -37,6 +37,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; +import org.junit.Before; public class TypeFieldTypeTests extends FieldTypeTestCase { @Override @@ -44,6 +45,17 @@ public class TypeFieldTypeTests extends FieldTypeTestCase { return new TypeFieldMapper.TypeFieldType(); } + @Before + public void setupProperties() { + addModifier(new Modifier("fielddata", true) { + @Override + public void modify(MappedFieldType ft) { + TypeFieldMapper.TypeFieldType tft = (TypeFieldMapper.TypeFieldType) ft; + tft.setFielddata(tft.fielddata() == false); + } + }); + } + public void testTermQuery() throws Exception { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig()); diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc index ae283e44569..703ee9930d2 100644 --- a/docs/reference/mapping/fields/type-field.asciidoc +++ b/docs/reference/mapping/fields/type-field.asciidoc @@ -5,7 +5,8 @@ Each document indexed is associated with a <> (see <>) and an <>. The `_type` field is indexed in order to make searching by type name fast. -The value of the `_type` field is accessible in queries and scripts: +The value of the `_type` field is accessible in queries, aggregations, +scripts, and when sorting: [source,js] -------------------------- @@ -27,9 +28,24 @@ GET my_index/type_*/_search "_type": [ "type_1", "type_2" ] <1> } }, + "aggs": { + "types": { + "terms": { + "field": "_type", <2> + "size": 10 + } + } + }, + "sort": [ + { + "_type": { <3> + "order": "desc" + } + } + ], "script_fields": { "type": { - "script": "doc['_type']" <2> + "script": "doc['_type']" <4> } } } @@ -38,4 +54,7 @@ GET my_index/type_*/_search // CONSOLE <1> Querying on the `_type` field -<2> Accessing the `_type` field in scripts (inline scripts must be <> for this example to work) +<2> Aggregating on the `_type` field +<3> Sorting on the `_type` field +<4> Accessing the `_type` field in scripts (inline scripts must be <> for this example to work) +