From 24a24d050ab312c98b91ee98a708f99f3d9f0a70 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 15 Sep 2020 20:24:10 -0400 Subject: [PATCH] Implement fields fetch for runtime fields (backport of #61995) (#62416) This implements the `fields` API in `_search` for runtime fields using doc values. Most of that implementation is stolen from the `docvalue_fields` fetch sub-phase, just moved into the same API that the `fields` API uses. At this point the `docvalue_fields` fetch phase looks like a special case of the `fields` API. While I was at it I moved the "which doc values sub-implementation should I use for fetching?" question from a bunch of `instanceof`s to a method on `LeafFieldData` so we can be much more flexible with what is returned and we're not forced to extend certain classes just to make the fetch phase happy. Relates to #59332 --- .../index/mapper/RankFeatureFieldMapper.java | 2 +- .../index/mapper/RankFeaturesFieldMapper.java | 2 +- .../index/mapper/ScaledFloatFieldMapper.java | 22 ++- .../mapper/SearchAsYouTypeFieldMapper.java | 7 +- .../index/mapper/TokenCountFieldMapper.java | 3 +- .../mapper/RankFeatureFieldMapperTests.java | 2 +- .../mapper/ScaledFloatFieldMapperTests.java | 2 +- .../test/scaled_float/10_basic.yml | 12 ++ .../join/mapper/MetaJoinFieldMapper.java | 2 +- .../join/mapper/ParentIdFieldMapper.java | 2 +- .../join/mapper/ParentJoinFieldMapper.java | 2 +- .../percolator/PercolatorFieldMapper.java | 3 +- .../PercolatorHighlightSubFetchPhase.java | 18 +- .../PercolatorMatchedSlotSubFetchPhase.java | 3 +- ...PercolatorHighlightSubFetchPhaseTests.java | 4 +- ...rcolatorMatchedSlotSubFetchPhaseTests.java | 13 +- .../ICUCollationKeywordFieldMapper.java | 2 +- .../ICUCollationKeywordFieldMapperTests.java | 2 +- .../AnnotatedTextFieldMapperTests.java | 2 +- .../AnnotatedTextFieldMapper.java | 3 +- .../mapper/murmur3/Murmur3FieldMapper.java | 2 +- .../test/search/10_source_filtering.yml | 17 +- .../test/search/240_date_nanos.yml | 11 +- .../search/fetch/FetchSubPhasePluginIT.java | 3 +- .../fielddata/IndexNumericFieldData.java | 4 +- .../index/fielddata/LeafFieldData.java | 26 +++ .../fielddata/plain/LeafDoubleFieldData.java | 26 ++- .../fielddata/plain/LeafLongFieldData.java | 28 ++- .../plain/SortedNumericIndexFieldData.java | 24 +++ .../mapper/AbstractGeometryFieldMapper.java | 3 +- .../index/mapper/BinaryFieldMapper.java | 2 +- .../index/mapper/BooleanFieldMapper.java | 2 +- .../index/mapper/CompletionFieldMapper.java | 3 +- .../index/mapper/DateFieldMapper.java | 2 +- .../index/mapper/DocValueFetcher.java | 79 ++++++++ .../index/mapper/FieldMapper.java | 4 +- .../index/mapper/IpFieldMapper.java | 2 +- .../index/mapper/KeywordFieldMapper.java | 2 +- .../index/mapper/MetadataFieldMapper.java | 3 +- .../index/mapper/NumberFieldMapper.java | 2 +- .../index/mapper/RangeFieldMapper.java | 2 +- .../index/mapper/TextFieldMapper.java | 6 +- .../index/mapper/ValueFetcher.java | 9 +- .../index/query/InnerHitContextBuilder.java | 3 +- .../index/query/QueryShardContext.java | 21 ++- .../search/DefaultSearchContext.java | 6 - .../elasticsearch/search/SearchService.java | 4 +- .../metrics/TopHitsAggregatorFactory.java | 3 +- .../search/fetch/FetchPhase.java | 45 +++-- .../search/fetch/FetchSubPhase.java | 16 +- .../search/fetch/subphase/ExplainPhase.java | 3 +- .../fetch/subphase/FetchDocValuesPhase.java | 178 +++--------------- .../fetch/subphase/FetchFieldsContext.java | 31 ++- .../fetch/subphase/FetchFieldsPhase.java | 16 +- .../fetch/subphase/FetchScorePhase.java | 3 +- .../fetch/subphase/FetchSourcePhase.java | 3 +- .../fetch/subphase/FetchVersionPhase.java | 3 +- .../fetch/subphase/FieldValueRetriever.java | 24 ++- .../search/fetch/subphase/InnerHitsPhase.java | 3 +- .../fetch/subphase/MatchedQueriesPhase.java | 3 +- .../fetch/subphase/ScriptFieldsPhase.java | 3 +- .../fetch/subphase/SeqNoPrimaryTermPhase.java | 3 +- .../subphase/highlight/HighlightPhase.java | 3 +- .../internal/FilteredSearchContext.java | 6 - .../search/internal/SearchContext.java | 3 - .../search/lookup/SourceLookup.java | 4 + .../index/mapper/BooleanFieldMapperTests.java | 2 +- .../mapper/CompletionFieldMapperTests.java | 2 +- .../index/mapper/DateFieldMapperTests.java | 29 ++- .../mapper/DocumentFieldMapperTests.java | 3 +- .../index/mapper/ExternalMapper.java | 3 +- .../index/mapper/FakeStringFieldMapper.java | 3 +- .../mapper/GeoPointFieldMapperTests.java | 2 +- .../mapper/GeoShapeFieldMapperTests.java | 2 +- .../index/mapper/IpFieldMapperTests.java | 2 +- .../index/mapper/IpRangeFieldMapperTests.java | 3 +- .../index/mapper/KeywordFieldMapperTests.java | 2 +- .../LegacyGeoShapeFieldMapperTests.java | 2 +- .../index/mapper/NumberFieldMapperTests.java | 2 +- .../index/mapper/ParametrizedMapperTests.java | 3 +- .../index/mapper/RangeFieldMapperTests.java | 4 +- .../index/mapper/TextFieldMapperTests.java | 2 +- .../fetch/subphase/FetchSourcePhaseTests.java | 5 +- .../subphase/FieldValueRetrieverTests.java | 10 +- .../index/mapper/FieldMapperTestCase.java | 6 +- .../index/mapper/MapperTestCase.java | 40 +++- .../index/mapper/MockFieldMapper.java | 3 +- .../aggregations/AggregatorTestCase.java | 7 - .../elasticsearch/test/TestSearchContext.java | 6 - .../mapper/HistogramFieldMapper.java | 2 +- .../search/AsyncSearchSingleNodeTests.java | 2 +- .../ConstantKeywordFieldMapperTests.java | 4 +- .../mapper/ConstantKeywordFieldMapper.java | 2 +- .../mapper/FlatObjectFieldMapperTests.java | 2 +- .../mapper/FlatObjectFieldMapper.java | 2 +- .../runtime-fields/qa/rest/build.gradle | 2 - .../mapper/RuntimeFieldMapper.java | 6 +- .../index/mapper/PointFieldMapperTests.java | 2 +- .../index/mapper/ShapeFieldMapperTests.java | 2 +- .../test/runtime_fields/10_keyword.yml | 14 ++ .../test/runtime_fields/20_long.yml | 26 +++ .../test/runtime_fields/30_double.yml | 19 ++ .../test/runtime_fields/40_date.yml | 23 +++ .../test/runtime_fields/50_ip.yml | 19 ++ .../test/runtime_fields/60_boolean.yml | 13 ++ .../mapper/DenseVectorFieldMapper.java | 2 +- .../mapper/SparseVectorFieldMapper.java | 2 +- .../wildcard/mapper/WildcardFieldMapper.java | 2 +- .../mapper/WildcardFieldMapperTests.java | 2 +- 109 files changed, 699 insertions(+), 349 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 3513406d2ae..705d3efd463 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -183,7 +183,7 @@ public class RankFeatureFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index 59b8ccb7f05..bf265e65c9f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -162,7 +162,7 @@ public class RankFeaturesFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index e4cd28b3705..ede3cbbc5b8 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -398,7 +398,7 @@ public class ScaledFloatFieldMapper extends ParametrizedFieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } @@ -545,5 +545,25 @@ public class ScaledFloatFieldMapper extends ParametrizedFieldMapper { } } + @Override + public DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + SortedNumericDoubleValues values = getDoubleValues(); + return new DocValueFetcher.Leaf() { + @Override + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); + } + + @Override + public int docValueCount() throws IOException { + return values.docValueCount(); + } + + @Override + public Object nextValue() throws IOException { + return format.format(values.nextValue()); + } + }; + } } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index 87ae686427f..53e06860af7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -54,6 +54,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -419,7 +420,7 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } @@ -465,7 +466,7 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } @@ -588,7 +589,7 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index 7e99fc1b196..087b828cdb2 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -25,6 +25,7 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.document.FieldType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Iterator; @@ -159,7 +160,7 @@ public class TokenCountFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index c5ff0fbc14b..870f6856eba 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -147,7 +147,7 @@ public class RankFeatureFieldMapperTests extends FieldMapperTestCase2() + new SearchHit( + slot, + "unknown", + new Text(hit.hit().getType()), + Collections.emptyMap(), + Collections.emptyMap() + ), + percolatorLeafReaderContext, + slot, + new SourceLookup(), + new HashMap<>() ); subContext.sourceLookup().setSource(document); // force source because MemoryIndex does not store fields diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java index 5d584099457..4b818e3a15a 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -57,7 +58,7 @@ final class PercolatorMatchedSlotSubFetchPhase implements FetchSubPhase { static final String FIELD_NAME_PREFIX = "_percolator_document_slot"; @Override - public FetchSubPhaseProcessor getProcessor(SearchContext searchContext) throws IOException { + public FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) throws IOException { List percolateContexts = new ArrayList<>(); List percolateQueries = locatePercolatorQuery(searchContext.query()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java index 15d5682725d..16f7030f14f 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java @@ -52,9 +52,9 @@ public class PercolatorHighlightSubFetchPhaseTests extends ESTestCase { Mockito.when(searchContext.highlight()).thenReturn(new SearchHighlightContext(Collections.emptyList())); Mockito.when(searchContext.query()).thenReturn(new MatchAllDocsQuery()); - assertNull(subFetchPhase.getProcessor(searchContext)); + assertNull(subFetchPhase.getProcessor(searchContext, null)); Mockito.when(searchContext.query()).thenReturn(percolateQuery); - assertNotNull(subFetchPhase.getProcessor(searchContext)); + assertNotNull(subFetchPhase.getProcessor(searchContext, null)); } public void testLocatePercolatorQuery() { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java index 26351f96dc0..bdc8db34d96 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import java.util.Collections; @@ -66,7 +67,7 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase { LeafReaderContext context = reader.leaves().get(0); // A match: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, new HashMap<>()); + HitContext hit = new HitContext(new SearchHit(0), context, 0, new SourceLookup(), new HashMap<>()); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); @@ -77,7 +78,7 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase { SearchContext sc = mock(SearchContext.class); when(sc.query()).thenReturn(percolateQuery); - FetchSubPhaseProcessor processor = phase.getProcessor(sc); + FetchSubPhaseProcessor processor = phase.getProcessor(sc, null); assertNotNull(processor); processor.process(hit); @@ -87,7 +88,7 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase { // No match: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, new HashMap<>()); + HitContext hit = new HitContext(new SearchHit(0), context, 0, new SourceLookup(), new HashMap<>()); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value1", new WhitespaceAnalyzer()); @@ -98,7 +99,7 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase { SearchContext sc = mock(SearchContext.class); when(sc.query()).thenReturn(percolateQuery); - FetchSubPhaseProcessor processor = phase.getProcessor(sc); + FetchSubPhaseProcessor processor = phase.getProcessor(sc, null); assertNotNull(processor); processor.process(hit); @@ -107,7 +108,7 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase { // No query: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, new HashMap<>()); + HitContext hit = new HitContext(new SearchHit(0), context, 0, new SourceLookup(), new HashMap<>()); PercolateQuery.QueryStore queryStore = ctx -> docId -> null; MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); @@ -118,7 +119,7 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase { SearchContext sc = mock(SearchContext.class); when(sc.query()).thenReturn(percolateQuery); - FetchSubPhaseProcessor processor = phase.getProcessor(sc); + FetchSubPhaseProcessor processor = phase.getProcessor(sc, null); assertNotNull(processor); processor.process(hit); diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 50d19c3c668..2e0b6764383 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -735,7 +735,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index e0ee73d3232..42fa7a660fb 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -310,7 +310,7 @@ public class ICUCollationKeywordFieldMapperTests extends FieldMapperTestCase2 extends Fie } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { String geoFormat = format != null ? format : GeoJsonGeometryFormat.NAME; AbstractGeometryFieldType mappedFieldType = fieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 8cc0d1fe556..959b911a49a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -193,7 +193,7 @@ public class BinaryFieldMapper extends ParametrizedFieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index ef799a8f5c3..dd5ccc64dc3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -254,7 +254,7 @@ public class BooleanFieldMapper extends ParametrizedFieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 0e356f85803..8726b217239 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -45,6 +45,7 @@ import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.suggest.completion.CompletionSuggester; import org.elasticsearch.search.suggest.completion.context.ContextMapping; import org.elasticsearch.search.suggest.completion.context.ContextMappings; @@ -536,7 +537,7 @@ public class CompletionFieldMapper extends ParametrizedFieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index a195eab3fb3..898765f2419 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -619,7 +619,7 @@ public final class DateFieldMapper extends ParametrizedFieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { DateFormatter defaultFormatter = fieldType().dateTimeFormatter(); DateFormatter formatter = format != null ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale()) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java new file mode 100644 index 00000000000..6dfa4021a68 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Collections.emptyList; + +/** + * Value fetcher that loads from doc values. + */ +public final class DocValueFetcher implements ValueFetcher { + private final DocValueFormat format; + private final IndexFieldData ifd; + private Leaf leaf; + + public DocValueFetcher(DocValueFormat format, IndexFieldData ifd) { + this.format = format; + this.ifd = ifd; + } + + public void setNextReader(LeafReaderContext context) { + leaf = ifd.load(context).getLeafValueFetcher(format); + } + + @Override + public List fetchValues(SourceLookup lookup) throws IOException { + if (false == leaf.advanceExact(lookup.docId())) { + return emptyList(); + } + List result = new ArrayList(leaf.docValueCount()); + for (int i = 0, count = leaf.docValueCount(); i < count; ++i) { + result.add(leaf.nextValue()); + } + return result; + } + + public interface Leaf { + /** + * Advance the doc values reader to the provided doc. + * @return false if there are no values for this document, true otherwise + */ + boolean advanceExact(int docId) throws IOException; + + /** + * A count of the number of values at this document. + */ + int docValueCount() throws IOException; + + /** + * Load and format the next value. + */ + Object nextValue() throws IOException; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index d5e1bf5386f..3129c59fddb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -35,6 +36,7 @@ import org.elasticsearch.common.xcontent.support.AbstractXContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType; import org.elasticsearch.search.fetch.subphase.FetchFieldsPhase; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -278,7 +280,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable { /** * Create a helper class to fetch field values during the {@link FetchFieldsPhase}. */ - public abstract ValueFetcher valueFetcher(MapperService mapperService, @Nullable String format); + public abstract ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, @Nullable String format); protected void createFieldNamesField(ParseContext context) { FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 78b35d2bb42..dcaf971d78e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -404,7 +404,7 @@ public class IpFieldMapper extends ParametrizedFieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 8b90d865615..99aa5251548 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -394,7 +394,7 @@ public final class KeywordFieldMapper extends ParametrizedFieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index f6a8b206283..6f59e891959 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Map; @@ -168,7 +169,7 @@ public abstract class MetadataFieldMapper extends ParametrizedFieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 4b1e599432f..f1074d6e0ea 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1089,7 +1089,7 @@ public class NumberFieldMapper extends ParametrizedFieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 22c8ba0b0e0..27c6a39ffa3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -368,7 +368,7 @@ public class RangeFieldMapper extends ParametrizedFieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { DateFormatter defaultFormatter = fieldType().dateTimeFormatter(); DateFormatter formatter = format != null ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale()) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 899fbde2bc2..e50e0455c39 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -503,7 +503,7 @@ public class TextFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } @@ -534,7 +534,7 @@ public class TextFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } @@ -841,7 +841,7 @@ public class TextFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java index 712be0ad351..7f8a5b61496 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java @@ -19,9 +19,11 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.search.fetch.subphase.FetchFieldsPhase; import org.elasticsearch.search.lookup.SourceLookup; +import java.io.IOException; import java.util.List; /** @@ -42,5 +44,10 @@ public interface ValueFetcher { * @param lookup a lookup structure over the document's source. * @return a list a standardized field values. */ - List fetchValues(SourceLookup lookup); + List fetchValues(SourceLookup lookup) throws IOException; + + /** + * Update the leaf reader used to fetch values. + */ + default void setNextReader(LeafReaderContext context) {} } diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java index 30f1cd36f59..f7f7bbf2c47 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java @@ -95,8 +95,7 @@ public abstract class InnerHitContextBuilder { } if (innerHitBuilder.getFetchFields() != null) { String indexName = queryShardContext.index().getName(); - FetchFieldsContext fieldsContext = FetchFieldsContext.create( - indexName, queryShardContext.getMapperService(), innerHitBuilder.getFetchFields()); + FetchFieldsContext fieldsContext = new FetchFieldsContext(innerHitBuilder.getFetchFields()); innerHitsContext.fetchFieldsContext(fieldsContext); } if (innerHitBuilder.getScriptFields() != null) { diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index a6915ebb8f3..843f0392041 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -325,16 +325,35 @@ public class QueryShardContext extends QueryRewriteContext { private SearchLookup lookup = null; + /** + * Get the lookup to use during the search. + */ public SearchLookup lookup() { if (this.lookup == null) { this.lookup = new SearchLookup( getMapperService(), (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup), - types); + types + ); } return this.lookup; } + /** + * Build a lookup customized for the fetch phase. Use {@link #lookup()} + * in other phases. + */ + public SearchLookup newFetchLookup() { + /* + * Real customization coming soon, I promise! + */ + return new SearchLookup( + getMapperService(), + (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup), + types + ); + } + public NestedScope nestedScope() { return nestedScope; } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index f2f93c03ee0..7943c5e6b7c 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -39,7 +39,6 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; @@ -511,11 +510,6 @@ final class DefaultSearchContext extends SearchContext { return indexService.cache().bitsetFilterCache(); } - @Override - public > IFD getForField(MappedFieldType fieldType) { - return queryShardContext.getForField(fieldType); - } - @Override public TimeValue timeout() { return timeout; diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index aa46fcc9b65..336dfeea091 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -968,9 +968,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv context.docValuesContext(docValuesContext); } if (source.fetchFields() != null) { - String indexName = context.indexShard().shardId().getIndexName(); - FetchFieldsContext fetchFieldsContext = FetchFieldsContext.create( - indexName, context.mapperService(), source.fetchFields()); + FetchFieldsContext fetchFieldsContext = new FetchFieldsContext(source.fetchFields()); context.fetchFieldsContext(fetchFieldsContext); } if (source.highlighter() != null) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java index 7b6c440d77a..42197d4277e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java @@ -114,8 +114,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory { subSearchContext.docValuesContext(docValuesContext); } if (fetchFields != null) { - String indexName = searchContext.indexShard().shardId().getIndexName(); - FetchFieldsContext fieldsContext = FetchFieldsContext.create(indexName, searchContext.mapperService(), fetchFields); + FetchFieldsContext fieldsContext = new FetchFieldsContext(fetchFields); subSearchContext.fetchFieldsContext(fieldsContext); } for (ScriptFieldsContext.ScriptField field : scriptFields) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index f55bf989aa9..27ab499a788 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -56,6 +56,7 @@ import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.InnerHitsPhase; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.tasks.TaskCancelledException; @@ -103,7 +104,8 @@ public class FetchPhase { SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()]; Map sharedCache = new HashMap<>(); - List processors = getProcessors(context); + SearchLookup lookup = context.getQueryShardContext().newFetchLookup(); + List processors = getProcessors(context, lookup); int currentReaderIndex = -1; LeafReaderContext currentReaderContext = null; @@ -122,8 +124,15 @@ public class FetchPhase { } } assert currentReaderContext != null; - HitContext hit - = prepareHitContext(context, fieldsVisitor, docId, storedToRequestedFields, currentReaderContext, sharedCache); + HitContext hit = prepareHitContext( + context, + lookup, + fieldsVisitor, + docId, + storedToRequestedFields, + currentReaderContext, + sharedCache + ); for (FetchSubPhaseProcessor processor : processors) { processor.process(hit); } @@ -141,11 +150,11 @@ public class FetchPhase { } - List getProcessors(SearchContext context) { + List getProcessors(SearchContext context, SearchLookup lookup) { try { List processors = new ArrayList<>(); for (FetchSubPhase fsp : fetchSubPhases) { - FetchSubPhaseProcessor processor = fsp.getProcessor(context); + FetchSubPhaseProcessor processor = fsp.getProcessor(context, lookup); if (processor != null) { processors.add(processor); } @@ -235,12 +244,20 @@ public class FetchPhase { return -1; } - private HitContext prepareHitContext(SearchContext context, FieldsVisitor fieldsVisitor, int docId, + private HitContext prepareHitContext(SearchContext context, SearchLookup lookup, FieldsVisitor fieldsVisitor, int docId, Map> storedToRequestedFields, LeafReaderContext subReaderContext, Map sharedCache) throws IOException { int rootDocId = findRootDocumentIfNested(context, subReaderContext, docId - subReaderContext.docBase); if (rootDocId == -1) { - return prepareNonNestedHitContext(context, fieldsVisitor, docId, storedToRequestedFields, subReaderContext, sharedCache); + return prepareNonNestedHitContext( + context, + lookup, + fieldsVisitor, + docId, + storedToRequestedFields, + subReaderContext, + sharedCache + ); } else { return prepareNestedHitContext(context, docId, rootDocId, storedToRequestedFields, subReaderContext, sharedCache); } @@ -254,6 +271,7 @@ public class FetchPhase { * fetch subphases that use the hit context to access the preloaded source. */ private HitContext prepareNonNestedHitContext(SearchContext context, + SearchLookup lookup, FieldsVisitor fieldsVisitor, int docId, Map> storedToRequestedFields, @@ -265,7 +283,7 @@ public class FetchPhase { if (fieldsVisitor == null) { SearchHit hit = new SearchHit(docId, null, typeText, null, null); - return new HitContext(hit, subReaderContext, subDocId, sharedCache); + return new HitContext(hit, subReaderContext, subDocId, lookup.source(), sharedCache); } else { SearchHit hit; loadStoredFields(context.mapperService(), subReaderContext, fieldsVisitor, subDocId); @@ -279,7 +297,7 @@ public class FetchPhase { hit = new SearchHit(docId, uid.id(), typeText, emptyMap(), emptyMap()); } - HitContext hitContext = new HitContext(hit, subReaderContext, subDocId, sharedCache); + HitContext hitContext = new HitContext(hit, subReaderContext, subDocId, lookup.source(), sharedCache); if (fieldsVisitor.source() != null) { hitContext.sourceLookup().setSource(fieldsVisitor.source()); } @@ -288,7 +306,6 @@ public class FetchPhase { } /** - /** * Resets the provided {@link HitContext} with information on the current * nested document. This includes the following: * - Adding an initial {@link SearchHit} instance. @@ -359,7 +376,13 @@ public class FetchPhase { getInternalNestedIdentity(context, nestedDocId, subReaderContext, context.mapperService(), nestedObjectMapper); SearchHit hit = new SearchHit(nestedTopDocId, rootId.id(), typeText, nestedIdentity, docFields, metaFields); - HitContext hitContext = new HitContext(hit, subReaderContext, nestedDocId, sharedCache); + HitContext hitContext = new HitContext( + hit, + subReaderContext, + nestedDocId, + new SourceLookup(), // Use a clean, fresh SourceLookup for the nested context + sharedCache + ); if (rootSourceAsMap != null) { // Isolate the nested json array object that matches with nested hit and wrap it back into the same json diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index dad4bf2dfce..4f549b20e8b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; @@ -38,14 +39,21 @@ public interface FetchSubPhase { private final SearchHit hit; private final LeafReaderContext readerContext; private final int docId; - private final SourceLookup sourceLookup = new SourceLookup(); + private final SourceLookup sourceLookup; private final Map cache; - public HitContext(SearchHit hit, LeafReaderContext context, int docId, Map cache) { + public HitContext( + SearchHit hit, + LeafReaderContext context, + int docId, + SourceLookup sourceLookup, + Map cache + ) { this.hit = hit; this.readerContext = context; this.docId = docId; - this.sourceLookup.setSegmentAndDocument(context, docId); + this.sourceLookup = sourceLookup; + sourceLookup.setSegmentAndDocument(context, docId); this.cache = cache; } @@ -95,5 +103,5 @@ public interface FetchSubPhase { * If nothing should be executed for the provided {@link SearchContext}, then the * implementation should return {@code null} */ - FetchSubPhaseProcessor getProcessor(SearchContext searchContext) throws IOException; + FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java index 010b9a0e128..f1ec50a7f4a 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java @@ -23,6 +23,7 @@ import org.apache.lucene.search.Explanation; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.rescore.RescoreContext; import java.io.IOException; @@ -33,7 +34,7 @@ import java.io.IOException; public final class ExplainPhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) { if (context.explain() == false || context.hasOnlySuggest()) { return null; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index b7eedf151cd..56e38b9c670 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -19,28 +19,20 @@ package org.elasticsearch.search.fetch.subphase; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; +import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Objects; - -import static org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; -import static org.elasticsearch.search.DocValueFormat.withNanosecondResolution; /** * Fetch sub phase which pulls data from doc values. @@ -53,7 +45,7 @@ public final class FetchDocValuesPhase implements FetchSubPhase { private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(FetchDocValuesPhase.class); @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) throws IOException { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) throws IOException { if (context.collapse() != null) { // retrieve the `doc_value` associated with the collapse field String name = context.collapse().getFieldName(); @@ -77,19 +69,27 @@ public final class FetchDocValuesPhase implements FetchSubPhase { "ease the transition to 7.x. It has become the default and shouldn't be set explicitly anymore."); } + /* + * Its tempting to swap this to a `Map` but that'd break backwards + * compatibility because we support fetching the same field multiple + * times with different configuration. That isn't possible with a `Map`. + */ List fields = new ArrayList<>(); for (FieldAndFormat fieldAndFormat : context.docValuesContext().fields()) { - DocValueField f = buildField(context, fieldAndFormat); - if (f != null) { - fields.add(f); + MappedFieldType ft = context.mapperService().fieldType(fieldAndFormat.field); + if (ft == null) { + continue; } + String format = USE_DEFAULT_FORMAT.equals(fieldAndFormat.format) ? null : fieldAndFormat.format; + ValueFetcher fetcher = new DocValueFetcher(ft.docValueFormat(format, null), lookup.doc().getForField(ft)); + fields.add(new DocValueField(fieldAndFormat.field, fetcher)); } return new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) throws IOException { for (DocValueField f : fields) { - f.setNextReader(readerContext); + f.fetcher.setNextReader(readerContext); } } @@ -103,153 +103,19 @@ public final class FetchDocValuesPhase implements FetchSubPhase { // docValues fields will still be document fields, and put under "fields" section of a hit. hit.hit().setDocumentField(f.field, hitField); } - f.setValues(hit.docId(), hitField); + hitField.getValues().addAll(f.fetcher.fetchValues(hit.sourceLookup())); } } }; } - private abstract static class DocValueField { + private class DocValueField { + private final String field; + private final ValueFetcher fetcher; - final String field; - final DocValueFormat format; - - protected DocValueField(String field, DocValueFormat format) { + DocValueField(String field, ValueFetcher fetcher) { this.field = field; - this.format = format; + this.fetcher = fetcher; } - - abstract void setNextReader(LeafReaderContext context); - abstract void setValues(int doc, DocumentField hitField) throws IOException; - - } - - private static class DoubleDocValueField extends DocValueField { - - SortedNumericDoubleValues doubleValues; - IndexNumericFieldData fieldData; - - DoubleDocValueField(String field, IndexNumericFieldData fieldData, DocValueFormat format) { - super(field, format); - this.fieldData = fieldData; - } - - @Override - void setNextReader(LeafReaderContext context) { - doubleValues = fieldData.load(context).getDoubleValues(); - } - - @Override - void setValues(int doc, DocumentField hitField) throws IOException { - final List values = hitField.getValues(); - if (doubleValues.advanceExact(doc)) { - for (int i = 0, count = doubleValues.docValueCount(); i < count; ++i) { - values.add(format.format(doubleValues.nextValue())); - } - } - } - } - - private static class NanoDocValueField extends DocValueField { - - SortedNumericDocValues longValues; - IndexNumericFieldData fieldData; - - NanoDocValueField(String field, IndexNumericFieldData fieldData, DocValueFormat format) { - super(field, withNanosecondResolution(format)); - this.fieldData = fieldData; - } - - @Override - void setNextReader(LeafReaderContext context) { - longValues = ((SortedNumericIndexFieldData.NanoSecondFieldData) fieldData.load(context)).getLongValuesAsNanos(); - } - - @Override - void setValues(int doc, DocumentField hitField) throws IOException { - final List values = hitField.getValues(); - if (longValues.advanceExact(doc)) { - for (int i = 0, count = longValues.docValueCount(); i < count; ++i) { - values.add(format.format(longValues.nextValue())); - } - } - } - } - - private static class LongDocValueField extends DocValueField { - - SortedNumericDocValues longValues; - IndexNumericFieldData fieldData; - - LongDocValueField(String field, IndexNumericFieldData fieldData, DocValueFormat format) { - super(field, format); - this.fieldData = fieldData; - } - - @Override - void setNextReader(LeafReaderContext context) { - longValues = fieldData.load(context).getLongValues(); - } - - @Override - void setValues(int doc, DocumentField hitField) throws IOException { - final List values = hitField.getValues(); - if (longValues.advanceExact(doc)) { - for (int i = 0, count = longValues.docValueCount(); i < count; ++i) { - values.add(format.format(longValues.nextValue())); - } - } - } - - } - - private static class BinaryDocValueField extends DocValueField { - - SortedBinaryDocValues byteValues; - IndexFieldData fieldData; - - BinaryDocValueField(String field, IndexFieldData fieldData, DocValueFormat format) { - super(field, format); - this.fieldData = fieldData; - } - - @Override - void setNextReader(LeafReaderContext context) { - byteValues = fieldData.load(context).getBytesValues(); - } - - @Override - void setValues(int doc, DocumentField hitField) throws IOException { - final List values = hitField.getValues(); - if (byteValues.advanceExact(doc)) { - for (int i = 0, count = byteValues.docValueCount(); i < count; ++i) { - values.add(format.format(byteValues.nextValue())); - } - } - } - - } - - private static DocValueField buildField(SearchContext context, FieldAndFormat fieldAndFormat) { - MappedFieldType fieldType = context.mapperService().fieldType(fieldAndFormat.field); - if (fieldType == null) { - return null; - } - final IndexFieldData indexFieldData = context.getForField(fieldType); - String formatDesc = fieldAndFormat.format; - if (Objects.equals(formatDesc, USE_DEFAULT_FORMAT)) { - formatDesc = null; - } - DocValueFormat format = fieldType.docValueFormat(formatDesc, null); - if (indexFieldData instanceof IndexNumericFieldData) { - if (((IndexNumericFieldData) indexFieldData).getNumericType().isFloatingPoint()) { - return new DoubleDocValueField(fieldAndFormat.field, (IndexNumericFieldData) indexFieldData, format); - } - if (((IndexNumericFieldData) indexFieldData).getNumericType() == NumericType.DATE_NANOSECONDS) { - return new NanoDocValueField(fieldAndFormat.field, (IndexNumericFieldData) indexFieldData, format); - } - return new LongDocValueField(fieldAndFormat.field, (IndexNumericFieldData) indexFieldData, format); - } - return new BinaryDocValueField(fieldAndFormat.field, indexFieldData, format); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java index 79b33003da7..d28fa314b5d 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java @@ -18,8 +18,8 @@ */ package org.elasticsearch.search.fetch.subphase; -import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.search.lookup.SearchLookup; import java.util.List; @@ -27,27 +27,18 @@ import java.util.List; * The context needed to retrieve fields. */ public class FetchFieldsContext { + private final List fields; - private FieldValueRetriever fieldValueRetriever; + public FetchFieldsContext(List fields) { + this.fields = fields; + } - public static FetchFieldsContext create(String indexName, - MapperService mapperService, - List fields) { - DocumentMapper documentMapper = mapperService.documentMapper(); - if (documentMapper.sourceMapper().enabled() == false) { - throw new IllegalArgumentException("Unable to retrieve the requested [fields] since _source is " + - "disabled in the mappings for index [" + indexName + "]"); + public FieldValueRetriever fieldValueRetriever(String indexName, MapperService mapperService, SearchLookup searchLookup) { + if (mapperService.documentMapper().sourceMapper().enabled() == false) { + throw new IllegalArgumentException( + "Unable to retrieve the requested [fields] since _source is disabled in the mappings for index [" + indexName + "]" + ); } - - FieldValueRetriever fieldValueRetriever = FieldValueRetriever.create(mapperService, fields); - return new FetchFieldsContext(fieldValueRetriever); - } - - private FetchFieldsContext(FieldValueRetriever fieldValueRetriever) { - this.fieldValueRetriever = fieldValueRetriever; - } - - public FieldValueRetriever fieldValueRetriever() { - return fieldValueRetriever; + return FieldValueRetriever.create(mapperService, searchLookup, fields); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index f4cab0c64b0..ae96a90fad1 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -26,8 +26,10 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; +import java.io.IOException; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -39,25 +41,29 @@ import java.util.Set; public final class FetchFieldsPhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext searchContext) { + public FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) { FetchFieldsContext fetchFieldsContext = searchContext.fetchFieldsContext(); if (fetchFieldsContext == null) { return null; } + FieldValueRetriever retriever = fetchFieldsContext.fieldValueRetriever( + searchContext.indexShard().shardId().getIndexName(), + searchContext.mapperService(), + lookup + ); return new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) { - + retriever.setNextReader(readerContext); } @Override - public void process(HitContext hitContext) { + public void process(HitContext hitContext) throws IOException { SearchHit hit = hitContext.hit(); SourceLookup sourceLookup = hitContext.sourceLookup(); - FieldValueRetriever fieldValueRetriever = fetchFieldsContext.fieldValueRetriever(); Set ignoredFields = getIgnoredFields(hit); - Map documentFields = fieldValueRetriever.retrieve(sourceLookup, ignoredFields); + Map documentFields = retriever.retrieve(sourceLookup, ignoredFields); for (Map.Entry entry : documentFields.entrySet()) { hit.setDocumentField(entry.getKey(), entry.getValue()); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java index 8f59a771da5..e9dd4291915 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java @@ -28,13 +28,14 @@ import org.apache.lucene.search.Weight; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; public class FetchScorePhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) throws IOException { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) throws IOException { if (context.trackScores() == false || context.docIdsToLoadSize() == 0 || // scores were already computed since they are needed on the coordinated node to merge top hits context.sort() == null) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java index 9801f6d5778..56802099bb9 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java @@ -28,6 +28,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; @@ -36,7 +37,7 @@ import java.util.Map; public final class FetchSourcePhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext searchContext) { + public FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) { if (searchContext.sourceRequested() == false) { return null; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java index 394d534b56e..0c60b91b5ec 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java @@ -25,13 +25,14 @@ import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; public final class FetchVersionPhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) { if (context.version() == false || (context.storedFieldsContext() != null && context.storedFieldsContext().fetchFields() == false)) { return null; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java index b6853a5a051..cea9137ac97 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.fetch.subphase; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.index.mapper.FieldAliasMapper; import org.elasticsearch.index.mapper.FieldMapper; @@ -26,8 +27,10 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -40,10 +43,11 @@ import java.util.Set; * Then given a specific document, it can retrieve the corresponding fields from the document's source. */ public class FieldValueRetriever { - private final List fieldContexts; - - public static FieldValueRetriever create(MapperService mapperService, - Collection fieldAndFormats) { + public static FieldValueRetriever create( + MapperService mapperService, + SearchLookup searchLookup, + Collection fieldAndFormats + ) { MappingLookup fieldMappers = mapperService.documentMapper().mappers(); List fieldContexts = new ArrayList<>(); @@ -65,7 +69,7 @@ public class FieldValueRetriever { } FieldMapper fieldMapper = (FieldMapper) mapper; - ValueFetcher valueFetcher = fieldMapper.valueFetcher(mapperService, format); + ValueFetcher valueFetcher = fieldMapper.valueFetcher(mapperService, searchLookup, format); fieldContexts.add(new FieldContext(field, valueFetcher)); } } @@ -73,11 +77,13 @@ public class FieldValueRetriever { return new FieldValueRetriever(fieldContexts); } + private final List fieldContexts; + private FieldValueRetriever(List fieldContexts) { this.fieldContexts = fieldContexts; } - public Map retrieve(SourceLookup sourceLookup, Set ignoredFields) { + public Map retrieve(SourceLookup sourceLookup, Set ignoredFields) throws IOException { Map documentFields = new HashMap<>(); for (FieldContext context : fieldContexts) { String field = context.fieldName; @@ -95,6 +101,12 @@ public class FieldValueRetriever { return documentFields; } + public void setNextReader(LeafReaderContext readerContext) { + for (FieldContext field : fieldContexts) { + field.valueFetcher.setNextReader(readerContext); + } + } + private static class FieldContext { final String fieldName; final ValueFetcher valueFetcher; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java index eb077a256b6..cc875ba196e 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java @@ -31,6 +31,7 @@ import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; @@ -46,7 +47,7 @@ public final class InnerHitsPhase implements FetchSubPhase { } @Override - public FetchSubPhaseProcessor getProcessor(SearchContext searchContext) { + public FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) { if (searchContext.innerHits() == null) { return null; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java index 3ee32dd299f..38499f69eb6 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -38,7 +39,7 @@ import java.util.Map; public final class MatchedQueriesPhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) throws IOException { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) throws IOException { if (context.docIdsToLoadSize() == 0 || // in case the request has only suggest, parsed query is null context.parsedQuery() == null) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java index 4145b1d4e39..082f99be580 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java @@ -25,6 +25,7 @@ import org.elasticsearch.script.FieldScript; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -35,7 +36,7 @@ import java.util.List; public final class ScriptFieldsPhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) { if (context.hasScriptFields() == false) { return null; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java index 5ed4a6ca9b6..b0862573915 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java @@ -25,13 +25,14 @@ import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; public final class SeqNoPrimaryTermPhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) throws IOException { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) throws IOException { if (context.seqNoAndPrimaryTerm() == false) { return null; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java index 4c169bc2d21..decc433f196 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java @@ -31,6 +31,7 @@ import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collection; @@ -49,7 +50,7 @@ public class HighlightPhase implements FetchSubPhase { } @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) { if (context.highlight() == null) { return null; } diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 95d15d5dded..3ee4878a4e5 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -27,7 +27,6 @@ import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; @@ -244,11 +243,6 @@ public abstract class FilteredSearchContext extends SearchContext { return in.bitsetFilterCache(); } - @Override - public > IFD getForField(MappedFieldType fieldType) { - return in.getForField(fieldType); - } - @Override public TimeValue timeout() { return in.timeout(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 3bbf3ea387f..11a477d3ebf 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; @@ -230,8 +229,6 @@ public abstract class SearchContext implements Releasable { public abstract BitsetFilterCache bitsetFilterCache(); - public abstract > IFD getForField(MappedFieldType fieldType); - public abstract TimeValue timeout(); public abstract void timeout(TimeValue timeout); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index a62b36dae38..9001d77eebb 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -55,6 +55,10 @@ public class SourceLookup implements Map { return sourceContentType; } + public int docId() { + return docId; + } + // Scripting requires this method to be public. Using source() // is not possible because certain checks use source == null as // as a determination if source is enabled/disabled, but it should diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 7957faee83b..6b857da556a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -172,7 +172,7 @@ public class BooleanFieldMapperTests extends MapperTestCase { assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), ft.termQuery("true", null)); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 4198f89b92b..9d2c3653a34 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -721,7 +721,7 @@ public class CompletionFieldMapperTests extends MapperTestCase { "The maximum allowed number of completion contexts in a mapping will be limited to [10] starting in version [8.0]."); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 20317a52c2a..77f908ac831 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; import org.elasticsearch.index.termvectors.TermVectorsService; +import org.elasticsearch.search.DocValueFormat; import java.io.IOException; import java.time.ZoneId; @@ -284,7 +285,7 @@ public class DateFieldMapperTests extends MapperTestCase { assertThat(e.getMessage(), containsString("Error parsing [format] on field [field]: Invalid")); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { DateFieldMapper mapper = createMapper(Resolution.MILLISECONDS, null); String date = "2020-05-15T21:33:02.000Z"; assertEquals(List.of(date), fetchSourceValue(mapper, date)); @@ -305,7 +306,7 @@ public class DateFieldMapperTests extends MapperTestCase { assertEquals(List.of(nullValueDate), fetchSourceValue(nullValueMapper, null)); } - public void testParseSourceValueWithFormat() { + public void testParseSourceValueWithFormat() throws IOException { DateFieldMapper mapper = createMapper(Resolution.NANOSECONDS, "strict_date_time", "1970-12-29T00:00:00.000Z"); String date = "1990-12-29T00:00:00.000Z"; assertEquals(List.of("1990/12/29"), fetchSourceValue(mapper, date, "yyyy/MM/dd")); @@ -313,7 +314,7 @@ public class DateFieldMapperTests extends MapperTestCase { assertEquals(List.of("1970/12/29"), fetchSourceValue(mapper, null, "yyyy/MM/dd")); } - public void testParseSourceValueNanos() { + public void testParseSourceValueNanos() throws IOException { DateFieldMapper mapper = createMapper(Resolution.NANOSECONDS, "strict_date_time||epoch_millis"); String date = "2020-05-15T21:33:02.123456789Z"; assertEquals(List.of("2020-05-15T21:33:02.123456789Z"), fetchSourceValue(mapper, date)); @@ -324,6 +325,28 @@ public class DateFieldMapperTests extends MapperTestCase { assertEquals(List.of(nullValueDate), fetchSourceValue(nullValueMapper, null)); } + public void testFetchDocValuesMillis() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "date").field("format", "strict_date_time||epoch_millis")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + DocValueFormat format = ft.docValueFormat(null, null); + String date = "2020-05-15T21:33:02.123Z"; + assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, date)); + assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, 1589578382123L)); + } + + public void testFetchDocValuesNanos() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "date_nanos").field("format", "strict_date_time||epoch_millis")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + DocValueFormat format = ft.docValueFormat(null, null); + String date = "2020-05-15T21:33:02.123456789Z"; + assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, date)); + assertEquals(List.of("2020-05-15T21:33:02.123Z"), fetchFromDocValues(mapperService, ft, format, 1589578382123L)); + } + private DateFieldMapper createMapper(Resolution resolution, String format) { return createMapper(resolution, format, null); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index edc40211c28..f6565ac25ec 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -32,6 +32,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.StringReader; @@ -103,7 +104,7 @@ public class DocumentFieldMapperTests extends LuceneTestCase { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index edc3233e398..40d8929733b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.geo.builders.PointBuilder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.geometry.Point; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.nio.charset.Charset; @@ -205,7 +206,7 @@ public class ExternalMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { @Override protected Object parseSourceValue(Object value) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 6c214fbc8c6..e7002668c54 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -31,6 +31,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; @@ -134,7 +135,7 @@ public class FakeStringFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { @Override protected String parseSourceValue(Object value) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 68257bb7bec..d90553d5f2d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -299,7 +299,7 @@ public class GeoPointFieldMapperTests extends FieldMapperTestCase2()); + HitContext hitContext = new HitContext(searchHit, leafReaderContext, 1, new SourceLookup(), new HashMap<>()); hitContext.sourceLookup().setSource(source == null ? null : BytesReference.bytes(source)); FetchSourcePhase phase = new FetchSourcePhase(); - FetchSubPhaseProcessor processor = phase.getProcessor(searchContext); + FetchSubPhaseProcessor processor = phase.getProcessor(searchContext, null); if (fetchSource == false) { assertNull(processor); } else { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java index 66babb2187a..a3cdff94df9 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java @@ -359,16 +359,20 @@ public class FieldValueRetrieverTests extends ESSingleNodeTestCase { assertFalse(fields.containsKey("object")); } - private Map retrieveFields(MapperService mapperService, XContentBuilder source, String fieldPattern) { + private Map retrieveFields(MapperService mapperService, XContentBuilder source, String fieldPattern) + throws IOException { + List fields = org.elasticsearch.common.collect.List.of(new FieldAndFormat(fieldPattern, null)); return retrieveFields(mapperService, source, fields); } - private Map retrieveFields(MapperService mapperService, XContentBuilder source, List fields) { + private Map retrieveFields(MapperService mapperService, XContentBuilder source, List fields) + throws IOException { + SourceLookup sourceLookup = new SourceLookup(); sourceLookup.setSource(BytesReference.bytes(source)); - FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, fields); + FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, null, fields); return fetchFieldsLookup.retrieve(sourceLookup, org.elasticsearch.common.collect.Set.of()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java index 225aae6748c..575ff5757de 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java @@ -254,17 +254,17 @@ public abstract class FieldMapperTestCase> exte return Strings.toString(x); } - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue) { + public static List fetchSourceValue(FieldMapper mapper, Object sourceValue) throws IOException { return fetchSourceValue(mapper, sourceValue, null); } - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue, String format) { + public static List fetchSourceValue(FieldMapper mapper, Object sourceValue, String format) throws IOException { String field = mapper.name(); MapperService mapperService = mock(MapperService.class); when(mapperService.sourcePath(field)).thenReturn(org.elasticsearch.common.collect.Set.of(field)); - ValueFetcher fetcher = mapper.valueFetcher(mapperService, format); + ValueFetcher fetcher = mapper.valueFetcher(mapperService, null, format); SourceLookup lookup = new SourceLookup(); lookup.setSource(Collections.singletonMap(field, sourceValue)); return fetcher.fetchValues(lookup); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 9190185daa4..a02e905a86f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -19,17 +19,27 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.function.BiFunction; +import java.util.function.Supplier; import static org.hamcrest.Matchers.containsString; import static org.mockito.Mockito.mock; @@ -121,19 +131,43 @@ public abstract class MapperTestCase extends MapperServiceTestCase { ); } - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue) { + public static List fetchSourceValue(FieldMapper mapper, Object sourceValue) throws IOException { return fetchSourceValue(mapper, sourceValue, null); } - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue, String format) { + public static List fetchSourceValue(FieldMapper mapper, Object sourceValue, String format) throws IOException { String field = mapper.name(); MapperService mapperService = mock(MapperService.class); when(mapperService.sourcePath(field)).thenReturn(org.elasticsearch.common.collect.Set.of(field)); - ValueFetcher fetcher = mapper.valueFetcher(mapperService, format); + ValueFetcher fetcher = mapper.valueFetcher(mapperService, null, format); SourceLookup lookup = new SourceLookup(); lookup.setSource(Collections.singletonMap(field, sourceValue)); return fetcher.fetchValues(lookup); } + + /** + * Use a {@linkplain FieldMapper} to extract values from doc values. + */ + protected final List fetchFromDocValues(MapperService mapperService, MappedFieldType ft, DocValueFormat format, Object sourceValue) + throws IOException { + + BiFunction, IndexFieldData> fieldDataLookup = (mft, lookupSource) -> mft + .fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService(), mapperService); + SetOnce> result = new SetOnce<>(); + withLuceneIndex(mapperService, iw -> { + iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field(ft.name(), sourceValue))).rootDoc()); + }, iw -> { + SearchLookup lookup = new SearchLookup(mapperService, fieldDataLookup, null); + ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.doc().getForField(ft)); + IndexSearcher searcher = newSearcher(iw); + LeafReaderContext context = searcher.getIndexReader().leaves().get(0); + lookup.source().setSegmentAndDocument(context, 0); + valueFetcher.setNextReader(context); + result.set(valueFetcher.fetchValues(lookup.source())); + }); + return result.get(); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 408462dc6d8..ae4b99ebcd9 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -28,6 +28,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; @@ -90,7 +91,7 @@ public class MockFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 9ec2df01708..c4f8742b72e 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -101,7 +101,6 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.mapper.MapperRegistry; -import org.elasticsearch.mock.orig.Mockito; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; @@ -319,12 +318,6 @@ public abstract class AggregatorTestCase extends ESTestCase { IndexFieldDataService ifds = new IndexFieldDataService(indexSettings, new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() { }), circuitBreakerService, mapperService); - when(searchContext.getForField(Mockito.any(MappedFieldType.class))) - .thenAnswer(invocationOnMock -> ifds.getForField((MappedFieldType) invocationOnMock.getArguments()[0], - indexSettings.getIndex().getName(), - () -> { - throw new UnsupportedOperationException("search lookup not available"); - })); QueryShardContext queryShardContext = queryShardContextMock(contextIndexSearcher, mapperService, indexSettings, circuitBreakerService, bigArrays); when(searchContext.getQueryShardContext()).thenReturn(queryShardContext); diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 1a418d1aebf..c04b87ccf69 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; @@ -308,11 +307,6 @@ public class TestSearchContext extends SearchContext { return fixedBitSetFilterCache; } - @Override - public > IFD getForField(MappedFieldType fieldType) { - return queryShardContext.getForField(fieldType); - } - @Override public TimeValue timeout() { return TimeValue.ZERO; diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 3930a7c8818..46c4f4a6584 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -168,7 +168,7 @@ public class HistogramFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java index ea905535a00..9543ffcd686 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java @@ -118,7 +118,7 @@ public class AsyncSearchSingleNodeTests extends ESSingleNodeTestCase { public static final class SubFetchPhasePlugin extends Plugin implements SearchPlugin { @Override public List getFetchSubPhases(FetchPhaseConstructionContext context) { - return Collections.singletonList(searchContext -> new FetchSubPhaseProcessor() { + return Collections.singletonList((searchContext, lookup) -> new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) {} diff --git a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index 3a486b64b30..6cc5a39183a 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -115,7 +115,7 @@ public class ConstantKeywordFieldMapperTests extends FieldMapperTestCase2 b.field("type", "constant_keyword"))); FieldMapper fieldMapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); - ValueFetcher fetcher = fieldMapper.valueFetcher(mapperService, null); + ValueFetcher fetcher = fieldMapper.valueFetcher(mapperService, null, null); SourceLookup missingValueLookup = new SourceLookup(); SourceLookup nullValueLookup = new SourceLookup(); @@ -126,7 +126,7 @@ public class ConstantKeywordFieldMapperTests extends FieldMapperTestCase2 b.field("type", "constant_keyword").field("value", "foo"))); fieldMapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); - fetcher = fieldMapper.valueFetcher(mapperService, null); + fetcher = fieldMapper.valueFetcher(mapperService, null, null); assertEquals(List.of("foo"), fetcher.fetchValues(missingValueLookup)); assertEquals(List.of("foo"), fetcher.fetchValues(nullValueLookup)); diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 08147bd5ac9..c0953baf2b3 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -267,7 +267,7 @@ public class ConstantKeywordFieldMapper extends FieldMapper { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java b/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java index 2804d8d69b1..f006007f53f 100644 --- a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java +++ b/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java @@ -513,7 +513,7 @@ public class FlatObjectFieldMapperTests extends FieldMapperTestCase