honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Store offsets in index prefix fields when stored in the parent field (#29067) 

The index prefix field is normally indexed as docs-only, given that it cannot
be used in phrases.  However, in the case that the parent field has been indexed
with offsets, or has term-vector offsets, we should also store this in the index
prefix field for highlighting.

Note that this commit does not implement highlighting on prefix fields, but
rather ensures that future work can implement this without a backwards-break
in index data.

Closes #28994
This commit is contained in:
Alan Woodward 2018-03-16 11:39:46 +00:00 committed by GitHub
parent 42c7c75298
commit 986e518170
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 89 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.analysis.AnalyzerWrapper;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
@ -152,11 +153,20 @@ public class TextFieldMapper extends FieldMapper {
fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap));
}
setupFieldType(context);
if (prefixFieldType != null && fieldType().isSearchable() == false) {
throw new IllegalArgumentException("Cannot set index_prefix on unindexed field [" + name() + "]");
PrefixFieldMapper prefixMapper = null;
if (prefixFieldType != null) {
if (fieldType().isSearchable() == false) {
throw new IllegalArgumentException("Cannot set index_prefix on unindexed field [" + name() + "]");
}
if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
prefixFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
}
if (fieldType.storeTermVectorOffsets()) {
prefixFieldType.setStoreTermVectorOffsets(true);
}
prefixFieldType.setAnalyzer(fieldType.indexAnalyzer());
prefixMapper = new PrefixFieldMapper(prefixFieldType, context.indexSettings());
}
PrefixFieldMapper prefixMapper = prefixFieldType == null ? null
: new PrefixFieldMapper(prefixFieldType.setAnalyzer(fieldType.indexAnalyzer()), context.indexSettings());
return new TextFieldMapper(
name, fieldType, defaultFieldType, positionIncrementGap, prefixMapper,
context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);

75
server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.mapper;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
@ -595,6 +596,80 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
assertThat(e.getMessage(), containsString("name cannot be empty string"));
}
public void testIndexPrefixIndexTypes() throws IOException {
QueryShardContext queryShardContext = indexService.newQueryShardContext(
randomInt(20), null, () -> {
throw new UnsupportedOperationException();
}, null);
{
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field")
.field("type", "text")
.field("analyzer", "english")
.startObject("index_prefix").endObject()
.field("index_options", "offsets")
.endObject().endObject().endObject().endObject().string();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix");
FieldType ft = prefix.fieldType;
assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, ft.indexOptions());
}
{
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field")
.field("type", "text")
.field("analyzer", "english")
.startObject("index_prefix").endObject()
.field("index_options", "positions")
.endObject().endObject().endObject().endObject().string();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix");
FieldType ft = prefix.fieldType;
assertEquals(IndexOptions.DOCS, ft.indexOptions());
assertFalse(ft.storeTermVectors());
}
{
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field")
.field("type", "text")
.field("analyzer", "english")
.startObject("index_prefix").endObject()
.field("term_vector", "with_positions_offsets")
.endObject().endObject().endObject().endObject().string();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix");
FieldType ft = prefix.fieldType;
assertEquals(IndexOptions.DOCS, ft.indexOptions());
assertTrue(ft.storeTermVectorOffsets());
}
{
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field")
.field("type", "text")
.field("analyzer", "english")
.startObject("index_prefix").endObject()
.field("term_vector", "with_positions")
.endObject().endObject().endObject().endObject().string();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
FieldMapper prefix = mapper.mappers().getMapper("field._index_prefix");
FieldType ft = prefix.fieldType;
assertEquals(IndexOptions.DOCS, ft.indexOptions());
assertFalse(ft.storeTermVectorOffsets());
}
}
public void testIndexPrefixMapping() throws IOException {
QueryShardContext queryShardContext = indexService.newQueryShardContext(