From 986e518170e18ce8d540753be24c8020baf4cb08 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 16 Mar 2018 11:39:46 +0000 Subject: [PATCH] Store offsets in index prefix fields when stored in the parent field (#29067) The index prefix field is normally indexed as docs-only, given that it cannot be used in phrases. However, in the case that the parent field has been indexed with offsets, or has term-vector offsets, we should also store this in the index prefix field for highlighting. Note that this commit does not implement highlighting on prefix fields, but rather ensures that future work can implement this without a backwards-break in index data. Closes #28994 --- .../index/mapper/TextFieldMapper.java | 18 ++++- .../index/mapper/TextFieldMapperTests.java | 75 +++++++++++++++++++ 2 files changed, 89 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 799698ac776..e2f8eb4e64f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -24,6 +24,7 @@ import org.apache.lucene.analysis.AnalyzerWrapper; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; @@ -152,11 +153,20 @@ public class TextFieldMapper extends FieldMapper { fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap)); } setupFieldType(context); - if (prefixFieldType != null && fieldType().isSearchable() == false) { - throw new IllegalArgumentException("Cannot set index_prefix on unindexed field [" + name() + "]"); + PrefixFieldMapper prefixMapper = null; + if (prefixFieldType != null) { + if (fieldType().isSearchable() == false) { + throw new IllegalArgumentException("Cannot set index_prefix on unindexed field [" + name() + "]"); + } + if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { + prefixFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + } + if (fieldType.storeTermVectorOffsets()) { + prefixFieldType.setStoreTermVectorOffsets(true); + } + prefixFieldType.setAnalyzer(fieldType.indexAnalyzer()); + prefixMapper = new PrefixFieldMapper(prefixFieldType, context.indexSettings()); } - PrefixFieldMapper prefixMapper = prefixFieldType == null ? null - : new PrefixFieldMapper(prefixFieldType.setAnalyzer(fieldType.indexAnalyzer()), context.indexSettings()); return new TextFieldMapper( name, fieldType, defaultFieldType, positionIncrementGap, prefixMapper, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 37c82ccc946..e63ff69f509 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -595,6 +596,80 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase { assertThat(e.getMessage(), containsString("name cannot be empty string")); } + public void testIndexPrefixIndexTypes() throws IOException { + QueryShardContext queryShardContext = indexService.newQueryShardContext( + randomInt(20), null, () -> { + throw new UnsupportedOperationException(); + }, null); + + { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("field") + .field("type", "text") + .field("analyzer", "english") + .startObject("index_prefix").endObject() + .field("index_options", "offsets") + .endObject().endObject().endObject().endObject().string(); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + + FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix"); + FieldType ft = prefix.fieldType; + assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, ft.indexOptions()); + } + + { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("field") + .field("type", "text") + .field("analyzer", "english") + .startObject("index_prefix").endObject() + .field("index_options", "positions") + .endObject().endObject().endObject().endObject().string(); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + + FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix"); + FieldType ft = prefix.fieldType; + assertEquals(IndexOptions.DOCS, ft.indexOptions()); + assertFalse(ft.storeTermVectors()); + } + + { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("field") + .field("type", "text") + .field("analyzer", "english") + .startObject("index_prefix").endObject() + .field("term_vector", "with_positions_offsets") + .endObject().endObject().endObject().endObject().string(); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + + FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix"); + FieldType ft = prefix.fieldType; + assertEquals(IndexOptions.DOCS, ft.indexOptions()); + assertTrue(ft.storeTermVectorOffsets()); + } + + { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("field") + .field("type", "text") + .field("analyzer", "english") + .startObject("index_prefix").endObject() + .field("term_vector", "with_positions") + .endObject().endObject().endObject().endObject().string(); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + + FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix"); + FieldType ft = prefix.fieldType; + assertEquals(IndexOptions.DOCS, ft.indexOptions()); + assertFalse(ft.storeTermVectorOffsets()); + } + } + public void testIndexPrefixMapping() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext(