honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move the shared fetch cache to highlighting. (#65105) 

The cache is only used by highlighters, so it can be scoped to only the
highlighting context.
This commit is contained in:
Julie Tibshirani 2020-11-16 17:59:05 -08:00
parent 23a1c64347
commit 3974c3b066
11 changed files with 43 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
View File

@ -39,7 +39,6 @@ import org.elasticsearch.search.lookup.SourceLookup;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -104,9 +103,7 @@ final class PercolatorHighlightSubFetchPhase implements FetchSubPhase {
), ),
percolatorLeafReaderContext, percolatorLeafReaderContext,
slot, slot,
new SourceLookup(), new SourceLookup());
new HashMap<>()
);
subContext.sourceLookup().setSource(document); subContext.sourceLookup().setSource(document);
// force source because MemoryIndex does not store fields // force source because MemoryIndex does not store fields
SearchHighlightContext highlight = new SearchHighlightContext(fetchContext.highlight().fields(), true); SearchHighlightContext highlight = new SearchHighlightContext(fetchContext.highlight().fields(), true);

13
modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java
View File

@ -43,7 +43,6 @@ import org.elasticsearch.search.lookup.SourceLookup;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.stream.IntStream; import java.util.stream.IntStream;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
@ -69,9 +68,7 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase {
new SearchHit(0), new SearchHit(0),
context, context,
0, 0,
new SourceLookup(), new SourceLookup());
new HashMap<>()
);
PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value"));
MemoryIndex memoryIndex = new MemoryIndex(); MemoryIndex memoryIndex = new MemoryIndex();
memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); memoryIndex.addField("field", "value", new WhitespaceAnalyzer());
@ -96,9 +93,7 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase {
new SearchHit(0), new SearchHit(0),
context, context,
0, 0,
new SourceLookup(), new SourceLookup());
new HashMap<>()
);
PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value"));
MemoryIndex memoryIndex = new MemoryIndex(); MemoryIndex memoryIndex = new MemoryIndex();
memoryIndex.addField("field", "value1", new WhitespaceAnalyzer()); memoryIndex.addField("field", "value1", new WhitespaceAnalyzer());
@ -122,9 +117,7 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase {
new SearchHit(0), new SearchHit(0),
context, context,
0, 0,
new SourceLookup(), new SourceLookup());
new HashMap<>()
);
PercolateQuery.QueryStore queryStore = ctx -> docId -> null; PercolateQuery.QueryStore queryStore = ctx -> docId -> null;
MemoryIndex memoryIndex = new MemoryIndex(); MemoryIndex memoryIndex = new MemoryIndex();
memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); memoryIndex.addField("field", "value", new WhitespaceAnalyzer());

30
server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
View File

@ -119,7 +119,6 @@ public class FetchPhase {
FetchContext fetchContext = new FetchContext(context); FetchContext fetchContext = new FetchContext(context);
SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()]; SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()];
Map<String, Object> sharedCache = new HashMap<>();
List<FetchSubPhaseProcessor> processors = getProcessors(context.shardTarget(), fetchContext); List<FetchSubPhaseProcessor> processors = getProcessors(context.shardTarget(), fetchContext);
@ -160,9 +159,7 @@ public class FetchPhase {
docId, docId,
storedToRequestedFields, storedToRequestedFields,
currentReaderContext, currentReaderContext,
fieldReader, fieldReader);
sharedCache
);
for (FetchSubPhaseProcessor processor : processors) { for (FetchSubPhaseProcessor processor : processors) {
processor.process(hit); processor.process(hit);
} }
@ -280,8 +277,7 @@ public class FetchPhase {
int docId, int docId,
Map<String, Set<String>> storedToRequestedFields, Map<String, Set<String>> storedToRequestedFields,
LeafReaderContext subReaderContext, LeafReaderContext subReaderContext,
CheckedBiConsumer<Integer, FieldsVisitor, IOException> storedFieldReader, CheckedBiConsumer<Integer, FieldsVisitor, IOException> storedFieldReader) throws IOException {
Map<String, Object> sharedCache) throws IOException {
int rootDocId = findRootDocumentIfNested(context, subReaderContext, docId - subReaderContext.docBase); int rootDocId = findRootDocumentIfNested(context, subReaderContext, docId - subReaderContext.docBase);
if (rootDocId == -1) { if (rootDocId == -1) {
return prepareNonNestedHitContext( return prepareNonNestedHitContext(
@ -291,12 +287,9 @@ public class FetchPhase {
docId, docId,
storedToRequestedFields, storedToRequestedFields,
subReaderContext, subReaderContext,
storedFieldReader, storedFieldReader);
sharedCache
);
} else { } else {
return prepareNestedHitContext(context, docId, rootDocId, storedToRequestedFields, return prepareNestedHitContext(context, docId, rootDocId, storedToRequestedFields, subReaderContext, storedFieldReader);
subReaderContext, storedFieldReader, sharedCache);
} }
} }
@ -313,15 +306,14 @@ public class FetchPhase {
int docId, int docId,
Map<String, Set<String>> storedToRequestedFields, Map<String, Set<String>> storedToRequestedFields,
LeafReaderContext subReaderContext, LeafReaderContext subReaderContext,
CheckedBiConsumer<Integer, FieldsVisitor, IOException> fieldReader, CheckedBiConsumer<Integer, FieldsVisitor, IOException> fieldReader) throws IOException {
Map<String, Object> sharedCache) throws IOException {
int subDocId = docId - subReaderContext.docBase; int subDocId = docId - subReaderContext.docBase;
DocumentMapper documentMapper = context.mapperService().documentMapper(); DocumentMapper documentMapper = context.mapperService().documentMapper();
Text typeText = documentMapper.typeText(); Text typeText = documentMapper.typeText();
if (fieldsVisitor == null) { if (fieldsVisitor == null) {
SearchHit hit = new SearchHit(docId, null, typeText, null, null); SearchHit hit = new SearchHit(docId, null, typeText, null, null);
return new HitContext(hit, subReaderContext, subDocId, lookup.source(), sharedCache); return new HitContext(hit, subReaderContext, subDocId, lookup.source());
} else { } else {
SearchHit hit; SearchHit hit;
loadStoredFields(context.mapperService(), fieldReader, fieldsVisitor, subDocId); loadStoredFields(context.mapperService(), fieldReader, fieldsVisitor, subDocId);
@ -335,7 +327,7 @@ public class FetchPhase {
hit = new SearchHit(docId, uid.id(), typeText, emptyMap(), emptyMap()); hit = new SearchHit(docId, uid.id(), typeText, emptyMap(), emptyMap());
} }
HitContext hitContext = new HitContext(hit, subReaderContext, subDocId, lookup.source(), sharedCache); HitContext hitContext = new HitContext(hit, subReaderContext, subDocId, lookup.source());
if (fieldsVisitor.source() != null) { if (fieldsVisitor.source() != null) {
hitContext.sourceLookup().setSource(fieldsVisitor.source()); hitContext.sourceLookup().setSource(fieldsVisitor.source());
} }
@ -357,8 +349,8 @@ public class FetchPhase {
int rootDocId, int rootDocId,
Map<String, Set<String>> storedToRequestedFields, Map<String, Set<String>> storedToRequestedFields,
LeafReaderContext subReaderContext, LeafReaderContext subReaderContext,
CheckedBiConsumer<Integer, FieldsVisitor, IOException> storedFieldReader, CheckedBiConsumer<Integer, FieldsVisitor, IOException> storedFieldReader)
Map<String, Object> sharedCache) throws IOException { throws IOException {
// Also if highlighting is requested on nested documents we need to fetch the _source from the root document, // Also if highlighting is requested on nested documents we need to fetch the _source from the root document,
// otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail, // otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail,
// because the entire _source is only stored with the root document. // because the entire _source is only stored with the root document.
@ -419,9 +411,7 @@ public class FetchPhase {
hit, hit,
subReaderContext, subReaderContext,
nestedDocId, nestedDocId,
new SourceLookup(), // Use a clean, fresh SourceLookup for the nested context new SourceLookup()); // Use a clean, fresh SourceLookup for the nested context
sharedCache
);
if (rootSourceAsMap != null) { if (rootSourceAsMap != null) {
// Isolate the nested json array object that matches with nested hit and wrap it back into the same json // Isolate the nested json array object that matches with nested hit and wrap it back into the same json

11
server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
View File

@ -26,7 +26,6 @@ import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.search.lookup.SourceLookup;
import java.io.IOException; import java.io.IOException;
import java.util.Map;
/** /**
* Sub phase within the fetch phase used to fetch things *about* the documents like highlighting or matched queries. * Sub phase within the fetch phase used to fetch things *about* the documents like highlighting or matched queries.
@ -38,21 +37,18 @@ public interface FetchSubPhase {
private final LeafReaderContext readerContext; private final LeafReaderContext readerContext;
private final int docId; private final int docId;
private final SourceLookup sourceLookup; private final SourceLookup sourceLookup;
private final Map<String, Object> cache;
public HitContext( public HitContext(
SearchHit hit, SearchHit hit,
LeafReaderContext context, LeafReaderContext context,
int docId, int docId,
SourceLookup sourceLookup, SourceLookup sourceLookup
Map<String, Object> cache
) { ) {
this.hit = hit; this.hit = hit;
this.readerContext = context; this.readerContext = context;
this.docId = docId; this.docId = docId;
this.sourceLookup = sourceLookup; this.sourceLookup = sourceLookup;
sourceLookup.setSegmentAndDocument(context, docId); sourceLookup.setSegmentAndDocument(context, docId);
this.cache = cache;
} }
public SearchHit hit() { public SearchHit hit() {
@ -88,11 +84,6 @@ public interface FetchSubPhase {
public IndexReader topLevelReader() { public IndexReader topLevelReader() {
return ReaderUtil.getTopLevelContext(readerContext).reader(); return ReaderUtil.getTopLevelContext(readerContext).reader();
} }
// TODO move this into Highlighter
public Map<String, Object> cache() {
return cache;
}
} }
/** /**

6
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java
View File

@ -81,10 +81,10 @@ public class FastVectorHighlighter implements Highlighter {
Encoder encoder = field.fieldOptions().encoder().equals("html") ? Encoder encoder = field.fieldOptions().encoder().equals("html") ?
HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT; HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
if (!hitContext.cache().containsKey(CACHE_KEY)) { if (!fieldContext.cache.containsKey(CACHE_KEY)) {
hitContext.cache().put(CACHE_KEY, new HighlighterEntry()); fieldContext.cache.put(CACHE_KEY, new HighlighterEntry());
} }
HighlighterEntry cache = (HighlighterEntry) hitContext.cache().get(CACHE_KEY); HighlighterEntry cache = (HighlighterEntry) fieldContext.cache.get(CACHE_KEY);
FieldHighlightEntry entry = cache.fields.get(fieldType); FieldHighlightEntry entry = cache.fields.get(fieldType);
if (entry == null) { if (entry == null) {
FragListBuilder fragListBuilder; FragListBuilder fragListBuilder;

7
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FieldHighlightContext.java
View File

@ -23,6 +23,8 @@ import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchContext;
import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhase;
import java.util.Map;
public class FieldHighlightContext { public class FieldHighlightContext {
public final String fieldName; public final String fieldName;
@ -32,6 +34,7 @@ public class FieldHighlightContext {
public final FetchSubPhase.HitContext hitContext; public final FetchSubPhase.HitContext hitContext;
public final Query query; public final Query query;
public final boolean forceSource; public final boolean forceSource;
public final Map<String, Object> cache;
public FieldHighlightContext(String fieldName, public FieldHighlightContext(String fieldName,
SearchHighlightContext.Field field, SearchHighlightContext.Field field,
@ -39,7 +42,8 @@ public class FieldHighlightContext {
FetchContext context, FetchContext context,
FetchSubPhase.HitContext hitContext, FetchSubPhase.HitContext hitContext,
Query query, Query query,
boolean forceSource) { boolean forceSource,
Map<String, Object> cache) {
this.fieldName = fieldName; this.fieldName = fieldName;
this.field = field; this.field = field;
this.fieldType = fieldType; this.fieldType = fieldType;
@ -47,5 +51,6 @@ public class FieldHighlightContext {
this.hitContext = hitContext; this.hitContext = hitContext;
this.query = query; this.query = query;
this.forceSource = forceSource; this.forceSource = forceSource;
this.cache = cache;
} }
} }

10
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java
View File

@ -56,7 +56,10 @@ public class HighlightPhase implements FetchSubPhase {
} }
public FetchSubPhaseProcessor getProcessor(FetchContext context, SearchHighlightContext highlightContext, Query query) { public FetchSubPhaseProcessor getProcessor(FetchContext context, SearchHighlightContext highlightContext, Query query) {
Map<String, Function<HitContext, FieldHighlightContext>> contextBuilders = contextBuilders(context, highlightContext, query); Map<String, Object> sharedCache = new HashMap<>();
Map<String, Function<HitContext, FieldHighlightContext>> contextBuilders = contextBuilders(
context, highlightContext, query, sharedCache);
return new FetchSubPhaseProcessor() { return new FetchSubPhaseProcessor() {
@Override @Override
public void setNextReader(LeafReaderContext readerContext) { public void setNextReader(LeafReaderContext readerContext) {
@ -98,7 +101,8 @@ public class HighlightPhase implements FetchSubPhase {
private Map<String, Function<HitContext, FieldHighlightContext>> contextBuilders(FetchContext context, private Map<String, Function<HitContext, FieldHighlightContext>> contextBuilders(FetchContext context,
SearchHighlightContext highlightContext, SearchHighlightContext highlightContext,
Query query) { Query query,
Map<String, Object> sharedCache) {
Map<String, Function<HitContext, FieldHighlightContext>> builders = new LinkedHashMap<>(); Map<String, Function<HitContext, FieldHighlightContext>> builders = new LinkedHashMap<>();
for (SearchHighlightContext.Field field : highlightContext.fields()) { for (SearchHighlightContext.Field field : highlightContext.fields()) {
Highlighter highlighter = getHighlighter(field); Highlighter highlighter = getHighlighter(field);
@ -147,7 +151,7 @@ public class HighlightPhase implements FetchSubPhase {
boolean forceSource = highlightContext.forceSource(field); boolean forceSource = highlightContext.forceSource(field);
builders.put(fieldName, builders.put(fieldName,
hc -> new FieldHighlightContext(fieldType.name(), field, fieldType, context, hc, hc -> new FieldHighlightContext(fieldType.name(), field, fieldType, context, hc,
highlightQuery == null ? query : highlightQuery, forceSource)); highlightQuery == null ? query : highlightQuery, forceSource, sharedCache));
} }
} }
return builders; return builders;

6
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java
View File

@ -61,12 +61,12 @@ public class PlainHighlighter implements Highlighter {
Encoder encoder = field.fieldOptions().encoder().equals("html") ? HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT; Encoder encoder = field.fieldOptions().encoder().equals("html") ? HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
if (!hitContext.cache().containsKey(CACHE_KEY)) { if (!fieldContext.cache.containsKey(CACHE_KEY)) {
hitContext.cache().put(CACHE_KEY, new HashMap<>()); fieldContext.cache.put(CACHE_KEY, new HashMap<>());
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
Map<MappedFieldType, org.apache.lucene.search.highlight.Highlighter> cache = Map<MappedFieldType, org.apache.lucene.search.highlight.Highlighter> cache =
(Map<MappedFieldType, org.apache.lucene.search.highlight.Highlighter>) hitContext.cache().get(CACHE_KEY); (Map<MappedFieldType, org.apache.lucene.search.highlight.Highlighter>) fieldContext.cache.get(CACHE_KEY);
org.apache.lucene.search.highlight.Highlighter entry = cache.get(fieldType); org.apache.lucene.search.highlight.Highlighter entry = cache.get(fieldType);
if (entry == null) { if (entry == null) {

2
server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
View File

@ -62,7 +62,7 @@ public class UnifiedHighlighter implements Highlighter {
@Override @Override
public HighlightField highlight(FieldHighlightContext fieldContext) throws IOException { public HighlightField highlight(FieldHighlightContext fieldContext) throws IOException {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
Map<String, CustomUnifiedHighlighter> cache = (Map<String, CustomUnifiedHighlighter>) fieldContext.hitContext.cache() Map<String, CustomUnifiedHighlighter> cache = (Map<String, CustomUnifiedHighlighter>) fieldContext.cache
.computeIfAbsent(UnifiedHighlighter.class.getName(), k -> new HashMap<>()); .computeIfAbsent(UnifiedHighlighter.class.getName(), k -> new HashMap<>());
if (cache.containsKey(fieldContext.fieldName) == false) { if (cache.containsKey(fieldContext.fieldName) == false) {
cache.put(fieldContext.fieldName, buildHighlighter(fieldContext)); cache.put(fieldContext.fieldName, buildHighlighter(fieldContext));

5
server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java
View File

@ -34,7 +34,6 @@ import org.elasticsearch.test.ESTestCase;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.Map; import java.util.Map;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
@ -163,9 +162,7 @@ public class FetchSourcePhaseTests extends ESTestCase {
searchHit, searchHit,
leafReaderContext, leafReaderContext,
1, 1,
new SourceLookup(), new SourceLookup());
new HashMap<>()
);
hitContext.sourceLookup().setSource(source == null ? null : BytesReference.bytes(source)); hitContext.sourceLookup().setSource(source == null ? null : BytesReference.bytes(source));
FetchSourcePhase phase = new FetchSourcePhase(); FetchSourcePhase phase = new FetchSourcePhase();

4
server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighter.java
View File

@ -34,11 +34,11 @@ public class CustomHighlighter implements Highlighter {
@Override @Override
public HighlightField highlight(FieldHighlightContext fieldContext) { public HighlightField highlight(FieldHighlightContext fieldContext) {
SearchHighlightContext.Field field = fieldContext.field; SearchHighlightContext.Field field = fieldContext.field;
CacheEntry cacheEntry = (CacheEntry) fieldContext.hitContext.cache().get("test-custom"); CacheEntry cacheEntry = (CacheEntry) fieldContext.cache.get("test-custom");
final int docId = fieldContext.hitContext.readerContext().docBase + fieldContext.hitContext.docId(); final int docId = fieldContext.hitContext.readerContext().docBase + fieldContext.hitContext.docId();
if (cacheEntry == null) { if (cacheEntry == null) {
cacheEntry = new CacheEntry(); cacheEntry = new CacheEntry();
fieldContext.hitContext.cache().put("test-custom", cacheEntry); fieldContext.cache.put("test-custom", cacheEntry);
cacheEntry.docId = docId; cacheEntry.docId = docId;
cacheEntry.position = 1; cacheEntry.position = 1;
} else { } else {