honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

In the internal highlighter APIs, use the field type as opposed to the mapper. (#31039)

This commit is contained in:
Julie Tibshirani 2018-06-04 11:12:03 -07:00 committed by GitHub
parent 53357e7d14
commit 609de08126
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 109 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java
View File

@ -36,7 +36,7 @@ import org.apache.lucene.search.vectorhighlight.SingleFragListBuilder;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight.Field;
@ -71,9 +71,9 @@ public class FastVectorHighlighter implements Highlighter {
SearchContextHighlight.Field field = highlighterContext.field;
SearchContext context = highlighterContext.context;
FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;
FieldMapper mapper = highlighterContext.mapper;
MappedFieldType fieldType = highlighterContext.fieldType;
if (canHighlight(mapper) == false) {
if (canHighlight(fieldType) == false) {
throw new IllegalArgumentException("the field [" + highlighterContext.fieldName +
"] should be indexed with term vector with position offsets to be used with fast vector highlighter");
}
@ -87,7 +87,7 @@ public class FastVectorHighlighter implements Highlighter {
HighlighterEntry cache = (HighlighterEntry) hitContext.cache().get(CACHE_KEY);
try {
MapperHighlightEntry entry = cache.mappers.get(mapper);
FieldHighlightEntry entry = cache.fields.get(fieldType);
if (entry == null) {
FragListBuilder fragListBuilder;
BaseFragmentsBuilder fragmentsBuilder;
@ -97,37 +97,37 @@ public class FastVectorHighlighter implements Highlighter {
if (field.fieldOptions().numberOfFragments() == 0) {
fragListBuilder = new SingleFragListBuilder();
if (!forceSource && mapper.fieldType().stored()) {
fragmentsBuilder = new SimpleFragmentsBuilder(mapper, field.fieldOptions().preTags(),
if (!forceSource && fieldType.stored()) {
fragmentsBuilder = new SimpleFragmentsBuilder(fieldType, field.fieldOptions().preTags(),
field.fieldOptions().postTags(), boundaryScanner);
} else {
fragmentsBuilder = new SourceSimpleFragmentsBuilder(mapper, context,
fragmentsBuilder = new SourceSimpleFragmentsBuilder(fieldType, context,
field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
}
} else {
fragListBuilder = field.fieldOptions().fragmentOffset() == -1 ?
new SimpleFragListBuilder() : new SimpleFragListBuilder(field.fieldOptions().fragmentOffset());
if (field.fieldOptions().scoreOrdered()) {
if (!forceSource && mapper.fieldType().stored()) {
if (!forceSource && fieldType.stored()) {
fragmentsBuilder = new ScoreOrderFragmentsBuilder(field.fieldOptions().preTags(),
field.fieldOptions().postTags(), boundaryScanner);
} else {
fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(mapper, context,
fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(fieldType, context,
field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
}
} else {
if (!forceSource && mapper.fieldType().stored()) {
fragmentsBuilder = new SimpleFragmentsBuilder(mapper, field.fieldOptions().preTags(),
if (!forceSource && fieldType.stored()) {
fragmentsBuilder = new SimpleFragmentsBuilder(fieldType, field.fieldOptions().preTags(),
field.fieldOptions().postTags(), boundaryScanner);
} else {
fragmentsBuilder =
new SourceSimpleFragmentsBuilder(mapper, context, field.fieldOptions().preTags(),
new SourceSimpleFragmentsBuilder(fieldType, context, field.fieldOptions().preTags(),
field.fieldOptions().postTags(), boundaryScanner);
}
}
}
fragmentsBuilder.setDiscreteMultiValueHighlighting(termVectorMultiValue);
entry = new MapperHighlightEntry();
entry = new FieldHighlightEntry();
if (field.fieldOptions().requireFieldMatch()) {
/**
* we use top level reader to rewrite the query against all readers,
@ -152,7 +152,7 @@ public class FastVectorHighlighter implements Highlighter {
cache.fvh = new org.apache.lucene.search.vectorhighlight.FastVectorHighlighter();
}
CustomFieldQuery.highlightFilters.set(field.fieldOptions().highlightFilter());
cache.mappers.put(mapper, entry);
cache.fields.put(fieldType, entry);
}
final FieldQuery fieldQuery;
if (field.fieldOptions().requireFieldMatch()) {
@ -173,12 +173,12 @@ public class FastVectorHighlighter implements Highlighter {
// Only send matched fields if they were requested to save time.
if (field.fieldOptions().matchedFields() != null && !field.fieldOptions().matchedFields().isEmpty()) {
fragments = cache.fvh.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(),
mapper.fieldType().name(), field.fieldOptions().matchedFields(), fragmentCharSize,
fieldType.name(), field.fieldOptions().matchedFields(), fragmentCharSize,
numberOfFragments, entry.fragListBuilder, entry.fragmentsBuilder, field.fieldOptions().preTags(),
field.fieldOptions().postTags(), encoder);
} else {
fragments = cache.fvh.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(),
mapper.fieldType().name(), fragmentCharSize, numberOfFragments, entry.fragListBuilder,
fieldType.name(), fragmentCharSize, numberOfFragments, entry.fragListBuilder,
entry.fragmentsBuilder, field.fieldOptions().preTags(), field.fieldOptions().postTags(), encoder);
}
@ -193,7 +193,7 @@ public class FastVectorHighlighter implements Highlighter {
FieldFragList fieldFragList = new SimpleFieldFragList(-1 /*ignored*/);
fieldFragList.add(0, noMatchSize, Collections.<WeightedPhraseInfo>emptyList());
fragments = entry.fragmentsBuilder.createFragments(hitContext.reader(), hitContext.docId(),
mapper.fieldType().name(), fieldFragList, 1, field.fieldOptions().preTags(),
fieldType.name(), fieldFragList, 1, field.fieldOptions().preTags(),
field.fieldOptions().postTags(), encoder);
if (fragments != null && fragments.length > 0) {
return new HighlightField(highlighterContext.fieldName, Text.convertFromStringArray(fragments));
@ -209,9 +209,10 @@ public class FastVectorHighlighter implements Highlighter {
}
@Override
public boolean canHighlight(FieldMapper fieldMapper) {
return fieldMapper.fieldType().storeTermVectors() && fieldMapper.fieldType().storeTermVectorOffsets()
&& fieldMapper.fieldType().storeTermVectorPositions();
public boolean canHighlight(MappedFieldType fieldType) {
return fieldType.storeTermVectors()
&& fieldType.storeTermVectorOffsets()
&& fieldType.storeTermVectorPositions();
}
private static BoundaryScanner getBoundaryScanner(Field field) {
@ -244,7 +245,7 @@ public class FastVectorHighlighter implements Highlighter {
}
}
private class MapperHighlightEntry {
private class FieldHighlightEntry {
public FragListBuilder fragListBuilder;
public FragmentsBuilder fragmentsBuilder;
public FieldQuery noFieldMatchFieldQuery;
@ -253,6 +254,6 @@ public class FastVectorHighlighter implements Highlighter {
private class HighlighterEntry {
public org.apache.lucene.search.vectorhighlight.FastVectorHighlighter fvh;
public Map<FieldMapper, MapperHighlightEntry> mappers = new HashMap<>();
public Map<MappedFieldType, FieldHighlightEntry> fields = new HashMap<>();
}
}

8
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java
View File

@ -29,7 +29,7 @@ import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.index.analysis.CustomAnalyzer;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import java.util.Comparator;
import java.util.List;
@ -47,10 +47,10 @@ public final class FragmentBuilderHelper {
* Fixes problems with broken analysis chains if positions and offsets are messed up that can lead to
* {@link StringIndexOutOfBoundsException} in the {@link FastVectorHighlighter}
*/
public static WeightedFragInfo fixWeightedFragInfo(FieldMapper mapper, Field[] values, WeightedFragInfo fragInfo) {
public static WeightedFragInfo fixWeightedFragInfo(MappedFieldType fieldType, Field[] values, WeightedFragInfo fragInfo) {
assert fragInfo != null : "FragInfo must not be null";
assert mapper.fieldType().name().equals(values[0].name()) : "Expected FieldMapper for field " + values[0].name();
if (!fragInfo.getSubInfos().isEmpty() && containsBrokenAnalysis(mapper.fieldType().indexAnalyzer())) {
assert fieldType.name().equals(values[0].name()) : "Expected MappedFieldType for field " + values[0].name();
if (!fragInfo.getSubInfos().isEmpty() && containsBrokenAnalysis(fieldType.indexAnalyzer())) {
/* This is a special case where broken analysis like WDF is used for term-vector creation at index-time
* which can potentially mess up the offsets. To prevent a SAIIOBException we need to resort
* the fragments based on their offsets rather than using soley the positions as it is done in

24
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java
View File

@ -24,18 +24,16 @@ import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.SourceFieldMapper;
import org.elasticsearch.index.mapper.TextFieldMapper;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.internal.SearchContext;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
@ -71,8 +69,8 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
boolean fieldNameContainsWildcards = field.field().contains("*");
for (String fieldName : fieldNamesToHighlight) {
FieldMapper fieldMapper = getMapperForField(fieldName, context, hitContext);
if (fieldMapper == null) {
MappedFieldType fieldType = context.mapperService().fullName(fieldName);
if (fieldType == null) {
continue;
}
@ -85,8 +83,8 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
// If the field was explicitly given we assume that whoever issued the query knew
// what they were doing and try to highlight anyway.
if (fieldNameContainsWildcards) {
if (fieldMapper.fieldType().typeName().equals(TextFieldMapper.CONTENT_TYPE) == false &&
fieldMapper.fieldType().typeName().equals(KeywordFieldMapper.CONTENT_TYPE) == false) {
if (fieldType.typeName().equals(TextFieldMapper.CONTENT_TYPE) == false &&
fieldType.typeName().equals(KeywordFieldMapper.CONTENT_TYPE) == false) {
continue;
}
}
@ -104,10 +102,10 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
if (highlightQuery == null) {
highlightQuery = context.parsedQuery().query();
}
HighlighterContext highlighterContext = new HighlighterContext(fieldName, field, fieldMapper, context,
hitContext, highlightQuery);
HighlighterContext highlighterContext = new HighlighterContext(fieldName,
field, fieldType, context, hitContext, highlightQuery);
if ((highlighter.canHighlight(fieldMapper) == false) && fieldNameContainsWildcards) {
if ((highlighter.canHighlight(fieldType) == false) && fieldNameContainsWildcards) {
// if several fieldnames matched the wildcard then we want to skip those that we cannot highlight
continue;
}
@ -119,10 +117,4 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
}
hitContext.hit().highlightFields(highlightFields);
}
private FieldMapper getMapperForField(String fieldName, SearchContext searchContext, HitContext hitContext) {
DocumentMapper documentMapper = searchContext.mapperService().documentMapper(hitContext.hit().getType());
// TODO: no need to lookup the doc mapper with unambiguous field names? just look at the mapper service
return documentMapper.mappers().smartNameFieldMapper(fieldName);
}
}

16
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightUtils.java
View File

@ -22,7 +22,7 @@ import org.apache.lucene.search.highlight.DefaultEncoder;
import org.apache.lucene.search.highlight.Encoder;
import org.apache.lucene.search.highlight.SimpleHTMLEncoder;
import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.lookup.SourceLookup;
@ -46,15 +46,17 @@ public final class HighlightUtils {
/**
* Load field values for highlighting.
*/
public static List<Object> loadFieldValues(SearchContextHighlight.Field field, FieldMapper mapper, SearchContext searchContext,
FetchSubPhase.HitContext hitContext) throws IOException {
public static List<Object> loadFieldValues(SearchContextHighlight.Field field,
MappedFieldType fieldType,
SearchContext searchContext,
FetchSubPhase.HitContext hitContext) throws IOException {
//percolator needs to always load from source, thus it sets the global force source to true
boolean forceSource = searchContext.highlight().forceSource(field);
List<Object> textsToHighlight;
if (!forceSource && mapper.fieldType().stored()) {
CustomFieldsVisitor fieldVisitor = new CustomFieldsVisitor(singleton(mapper.fieldType().name()), false);
if (!forceSource && fieldType.stored()) {
CustomFieldsVisitor fieldVisitor = new CustomFieldsVisitor(singleton(fieldType.name()), false);
hitContext.reader().document(hitContext.docId(), fieldVisitor);
textsToHighlight = fieldVisitor.fields().get(mapper.fieldType().name());
textsToHighlight = fieldVisitor.fields().get(fieldType.name());
if (textsToHighlight == null) {
// Can happen if the document doesn't have the field to highlight
textsToHighlight = Collections.emptyList();
@ -62,7 +64,7 @@ public final class HighlightUtils {
} else {
SourceLookup sourceLookup = searchContext.lookup().source();
sourceLookup.setSegmentAndDocument(hitContext.readerContext(), hitContext.docId());
textsToHighlight = sourceLookup.extractRawValues(mapper.fieldType().name());
textsToHighlight = sourceLookup.extractRawValues(fieldType.name());
}
assert textsToHighlight != null;
return textsToHighlight;

4
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/Highlighter.java
View File

@ -18,7 +18,7 @@
*/
package org.elasticsearch.search.fetch.subphase.highlight;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
/**
* Highlights a search result.
@ -27,5 +27,5 @@ public interface Highlighter {
HighlightField highlight(HighlighterContext highlighterContext);
boolean canHighlight(FieldMapper fieldMapper);
boolean canHighlight(MappedFieldType fieldType);
}

14
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterContext.java
View File

@ -19,7 +19,7 @@
package org.elasticsearch.search.fetch.subphase.highlight;
import org.apache.lucene.search.Query;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.internal.SearchContext;
@ -27,16 +27,20 @@ public class HighlighterContext {
public final String fieldName;
public final SearchContextHighlight.Field field;
public final FieldMapper mapper;
public final MappedFieldType fieldType;
public final SearchContext context;
public final FetchSubPhase.HitContext hitContext;
public final Query query;
public HighlighterContext(String fieldName, SearchContextHighlight.Field field, FieldMapper mapper, SearchContext context,
FetchSubPhase.HitContext hitContext, Query query) {
public HighlighterContext(String fieldName,
SearchContextHighlight.Field field,
MappedFieldType fieldType,
SearchContext context,
FetchSubPhase.HitContext hitContext,
Query query) {
this.fieldName = fieldName;
this.field = field;
this.mapper = mapper;
this.fieldType = fieldType;
this.context = context;
this.hitContext = hitContext;
this.query = query;

29
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
View File

@ -36,7 +36,7 @@ import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.internal.SearchContext;
@ -59,22 +59,21 @@ public class PlainHighlighter implements Highlighter {
SearchContextHighlight.Field field = highlighterContext.field;
SearchContext context = highlighterContext.context;
FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;
FieldMapper mapper = highlighterContext.mapper;
MappedFieldType fieldType = highlighterContext.fieldType;
Encoder encoder = field.fieldOptions().encoder().equals("html") ? HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
if (!hitContext.cache().containsKey(CACHE_KEY)) {
Map<FieldMapper, org.apache.lucene.search.highlight.Highlighter> mappers = new HashMap<>();
hitContext.cache().put(CACHE_KEY, mappers);
hitContext.cache().put(CACHE_KEY, new HashMap<>());
}
@SuppressWarnings("unchecked")
Map<FieldMapper, org.apache.lucene.search.highlight.Highlighter> cache =
(Map<FieldMapper, org.apache.lucene.search.highlight.Highlighter>) hitContext.cache().get(CACHE_KEY);
Map<MappedFieldType, org.apache.lucene.search.highlight.Highlighter> cache =
(Map<MappedFieldType, org.apache.lucene.search.highlight.Highlighter>) hitContext.cache().get(CACHE_KEY);
org.apache.lucene.search.highlight.Highlighter entry = cache.get(mapper);
org.apache.lucene.search.highlight.Highlighter entry = cache.get(fieldType);
if (entry == null) {
QueryScorer queryScorer = new CustomQueryScorer(highlighterContext.query,
field.fieldOptions().requireFieldMatch() ? mapper.fieldType().name() : null);
field.fieldOptions().requireFieldMatch() ? fieldType.name() : null);
queryScorer.setExpandMultiTermQuery(true);
Fragmenter fragmenter;
if (field.fieldOptions().numberOfFragments() == 0) {
@ -96,21 +95,21 @@ public class PlainHighlighter implements Highlighter {
// always highlight across all data
entry.setMaxDocCharsToAnalyze(Integer.MAX_VALUE);
cache.put(mapper, entry);
cache.put(fieldType, entry);
}
// a HACK to make highlighter do highlighting, even though its using the single frag list builder
int numberOfFragments = field.fieldOptions().numberOfFragments() == 0 ? 1 : field.fieldOptions().numberOfFragments();
ArrayList<TextFragment> fragsList = new ArrayList<>();
List<Object> textsToHighlight;
Analyzer analyzer = getAnalyzer(context.mapperService().documentMapper(hitContext.hit().getType()), mapper.fieldType());
Analyzer analyzer = getAnalyzer(context.mapperService().documentMapper(hitContext.hit().getType()), fieldType);
final int maxAnalyzedOffset = context.indexShard().indexSettings().getHighlightMaxAnalyzedOffset();
try {
textsToHighlight = HighlightUtils.loadFieldValues(field, mapper, context, hitContext);
textsToHighlight = HighlightUtils.loadFieldValues(field, fieldType, context, hitContext);
for (Object textToHighlight : textsToHighlight) {
String text = convertFieldValue(mapper.fieldType(), textToHighlight);
String text = convertFieldValue(fieldType, textToHighlight);
if (text.length() > maxAnalyzedOffset) {
throw new IllegalArgumentException(
"The length of [" + highlighterContext.fieldName + "] field of [" + hitContext.hit().getId() +
@ -121,7 +120,7 @@ public class PlainHighlighter implements Highlighter {
"with unified or fvh highlighter is recommended!");
}
try (TokenStream tokenStream = analyzer.tokenStream(mapper.fieldType().name(), text)) {
try (TokenStream tokenStream = analyzer.tokenStream(fieldType.name(), text)) {
if (!tokenStream.hasAttribute(CharTermAttribute.class) || !tokenStream.hasAttribute(OffsetAttribute.class)) {
// can't perform highlighting if the stream has no terms (binary token stream) or no offsets
continue;
@ -178,7 +177,7 @@ public class PlainHighlighter implements Highlighter {
String fieldContents = textsToHighlight.get(0).toString();
int end;
try {
end = findGoodEndForNoHighlightExcerpt(noMatchSize, analyzer, mapper.fieldType().name(), fieldContents);
end = findGoodEndForNoHighlightExcerpt(noMatchSize, analyzer, fieldType.name(), fieldContents);
} catch (Exception e) {
throw new FetchPhaseExecutionException(context, "Failed to highlight field [" + highlighterContext.fieldName + "]", e);
}
@ -190,7 +189,7 @@ public class PlainHighlighter implements Highlighter {
}
@Override
public boolean canHighlight(FieldMapper fieldMapper) {
public boolean canHighlight(MappedFieldType fieldType) {
return true;
}

21
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SimpleFragmentsBuilder.java
View File

@ -23,24 +23,27 @@ import org.apache.lucene.search.highlight.Encoder;
import org.apache.lucene.search.vectorhighlight.BoundaryScanner;
import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
/**
* Direct Subclass of Lucene's org.apache.lucene.search.vectorhighlight.SimpleFragmentsBuilder
* that corrects offsets for broken analysis chains.
* Direct Subclass of Lucene's org.apache.lucene.search.vectorhighlight.SimpleFragmentsBuilder
* that corrects offsets for broken analysis chains.
*/
public class SimpleFragmentsBuilder extends org.apache.lucene.search.vectorhighlight.SimpleFragmentsBuilder {
protected final FieldMapper mapper;
protected final MappedFieldType fieldType;
public SimpleFragmentsBuilder(FieldMapper mapper,
String[] preTags, String[] postTags, BoundaryScanner boundaryScanner) {
public SimpleFragmentsBuilder(MappedFieldType fieldType,
String[] preTags,
String[] postTags,
BoundaryScanner boundaryScanner) {
super(preTags, postTags, boundaryScanner);
this.mapper = mapper;
this.fieldType = fieldType;
}
@Override
protected String makeFragment( StringBuilder buffer, int[] index, Field[] values, WeightedFragInfo fragInfo,
String[] preTags, String[] postTags, Encoder encoder ){
return super.makeFragment(buffer, index, values, FragmentBuilderHelper.fixWeightedFragInfo(mapper, values, fragInfo),
preTags, postTags, encoder);
WeightedFragInfo weightedFragInfo = FragmentBuilderHelper.fixWeightedFragInfo(fieldType, values, fragInfo);
return super.makeFragment(buffer, index, values, weightedFragInfo, preTags, postTags, encoder);
}
}

19
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceScoreOrderFragmentsBuilder.java
View File

@ -26,7 +26,7 @@ import org.apache.lucene.search.highlight.Encoder;
import org.apache.lucene.search.vectorhighlight.BoundaryScanner;
import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
import org.apache.lucene.search.vectorhighlight.ScoreOrderFragmentsBuilder;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.lookup.SourceLookup;
@ -35,14 +35,17 @@ import java.util.List;
public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder {
private final FieldMapper mapper;
private final MappedFieldType fieldType;
private final SearchContext searchContext;
public SourceScoreOrderFragmentsBuilder(FieldMapper mapper, SearchContext searchContext, String[] preTags, String[] postTags,
public SourceScoreOrderFragmentsBuilder(MappedFieldType fieldType,
SearchContext searchContext,
String[] preTags,
String[] postTags,
BoundaryScanner boundaryScanner) {
super(preTags, postTags, boundaryScanner);
this.mapper = mapper;
this.fieldType = fieldType;
this.searchContext = searchContext;
}
@ -52,10 +55,10 @@ public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder
SourceLookup sourceLookup = searchContext.lookup().source();
sourceLookup.setSegmentAndDocument((LeafReaderContext) reader.getContext(), docId);
List<Object> values = sourceLookup.extractRawValues(mapper.fieldType().name());
List<Object> values = sourceLookup.extractRawValues(fieldType.name());
Field[] fields = new Field[values.size()];
for (int i = 0; i < values.size(); i++) {
fields[i] = new Field(mapper.fieldType().name(), values.get(i).toString(), TextField.TYPE_NOT_STORED);
fields[i] = new Field(fieldType.name(), values.get(i).toString(), TextField.TYPE_NOT_STORED);
}
return fields;
}
@ -63,7 +66,7 @@ public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder
@Override
protected String makeFragment( StringBuilder buffer, int[] index, Field[] values, WeightedFragInfo fragInfo,
String[] preTags, String[] postTags, Encoder encoder ){
return super.makeFragment(buffer, index, values, FragmentBuilderHelper.fixWeightedFragInfo(mapper, values, fragInfo),
preTags, postTags, encoder);
WeightedFragInfo weightedFragInfo = FragmentBuilderHelper.fixWeightedFragInfo(fieldType, values, fragInfo);
return super.makeFragment(buffer, index, values, weightedFragInfo, preTags, postTags, encoder);
}
}

13
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceSimpleFragmentsBuilder.java
View File

@ -23,7 +23,7 @@ import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.vectorhighlight.BoundaryScanner;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.lookup.SourceLookup;
@ -34,9 +34,12 @@ public class SourceSimpleFragmentsBuilder extends SimpleFragmentsBuilder {
private final SearchContext searchContext;
public SourceSimpleFragmentsBuilder(FieldMapper mapper, SearchContext searchContext, String[] preTags, String[] postTags,
public SourceSimpleFragmentsBuilder(MappedFieldType fieldType,
SearchContext searchContext,
String[] preTags,
String[] postTags,
BoundaryScanner boundaryScanner) {
super(mapper, preTags, postTags, boundaryScanner);
super(fieldType, preTags, postTags, boundaryScanner);
this.searchContext = searchContext;
}
@ -48,13 +51,13 @@ public class SourceSimpleFragmentsBuilder extends SimpleFragmentsBuilder {
SourceLookup sourceLookup = searchContext.lookup().source();
sourceLookup.setSegmentAndDocument((LeafReaderContext) reader.getContext(), docId);
List<Object> values = sourceLookup.extractRawValues(mapper.fieldType().name());
List<Object> values = sourceLookup.extractRawValues(fieldType.name());
if (values.isEmpty()) {
return EMPTY_FIELDS;
}
Field[] fields = new Field[values.size()];
for (int i = 0; i < values.size(); i++) {
fields[i] = new Field(mapper.fieldType().name(), values.get(i).toString(), TextField.TYPE_NOT_STORED);
fields[i] = new Field(fieldType.name(), values.get(i).toString(), TextField.TYPE_NOT_STORED);
}
return fields;
}

15
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
View File

@ -22,11 +22,11 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.highlight.Encoder;
import org.apache.lucene.search.uhighlight.Snippet;
import org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner;
import org.apache.lucene.search.uhighlight.CustomPassageFormatter;
import org.apache.lucene.search.uhighlight.CustomSeparatorBreakIterator;
import org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter;
import org.apache.lucene.search.uhighlight.Snippet;
import org.apache.lucene.search.uhighlight.UnifiedHighlighter.OffsetSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CollectionUtil;
@ -34,7 +34,6 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
@ -52,13 +51,13 @@ import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTI
public class UnifiedHighlighter implements Highlighter {
@Override
public boolean canHighlight(FieldMapper fieldMapper) {
public boolean canHighlight(MappedFieldType fieldType) {
return true;
}
@Override
public HighlightField highlight(HighlighterContext highlighterContext) {
FieldMapper fieldMapper = highlighterContext.mapper;
MappedFieldType fieldType = highlighterContext.fieldType;
SearchContextHighlight.Field field = highlighterContext.field;
SearchContext context = highlighterContext.context;
FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;
@ -72,15 +71,15 @@ public class UnifiedHighlighter implements Highlighter {
try {
final Analyzer analyzer =
getAnalyzer(context.mapperService().documentMapper(hitContext.hit().getType()), fieldMapper.fieldType());
List<Object> fieldValues = HighlightUtils.loadFieldValues(field, fieldMapper, context, hitContext);
getAnalyzer(context.mapperService().documentMapper(hitContext.hit().getType()), fieldType);
List<Object> fieldValues = HighlightUtils.loadFieldValues(field, fieldType, context, hitContext);
fieldValues = fieldValues.stream()
.map((s) -> convertFieldValue(fieldMapper.fieldType(), s))
.map((s) -> convertFieldValue(fieldType, s))
.collect(Collectors.toList());
final IndexSearcher searcher = new IndexSearcher(hitContext.reader());
final CustomUnifiedHighlighter highlighter;
final String fieldValue = mergeFieldValues(fieldValues, MULTIVAL_SEP_CHAR);
final OffsetSource offsetSource = getOffsetSource(fieldMapper.fieldType());
final OffsetSource offsetSource = getOffsetSource(fieldType);
if ((offsetSource == OffsetSource.ANALYSIS) && (fieldValue.length() > maxAnalyzedOffset)) {
throw new IllegalArgumentException(
"The length of [" + highlighterContext.fieldName + "] field of [" + hitContext.hit().getId() +

8
server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighter.java
View File

@ -19,11 +19,7 @@
package org.elasticsearch.search.fetch.subphase.highlight;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
import org.elasticsearch.search.fetch.subphase.highlight.HighlighterContext;
import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
import org.elasticsearch.index.mapper.MappedFieldType;
import java.util.ArrayList;
import java.util.List;
@ -68,7 +64,7 @@ public class CustomHighlighter implements Highlighter {
}
@Override
public boolean canHighlight(FieldMapper fieldMapper) {
public boolean canHighlight(MappedFieldType fieldType) {
return true;
}