From f769821bc80652d4cb0237c18a78bd09abad38da Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 27 Aug 2020 18:09:56 +0200 Subject: [PATCH] Pass SearchLookup supplier through to fielddataBuilder (#61430) (#61638) Runtime fields need to have a SearchLookup available, when building their fielddata implementations, so that they can look up other fields, runtime or not. To achieve that, we add a Supplier argument to the existing MappedFieldType#fielddataBuilder method. As we introduce the ability to look up other fields while building fielddata for mapped fields, we implicitly add the ability for a field to require other fields. This requires some protection mechanism that detects dependency cycles to prevent stack overflow errors. With this commit we also introduce detection for cycles, as well as a limit on the depth of the references for a runtime field. Note that we also plan on introducing cycles detection at compile time, so the runtime cycles detection is a last resort to prevent stack overflow errors but we hope that we can reject runtime fields from being registered in the mappings when they create a cycle in their definition. Note that this commit does not introduce any production implementation of runtime fields, but is rather a pre-requisite to merge the runtime fields feature branch. This is a breaking change for MapperPlugins that plug in a mapper, as the signature of MappedFieldType#fielddataBuilder changes from taking a single argument (the index name), to also accept a Supplier. Relates to #59332 Co-authored-by: Nik Everett --- .../ExpressionFieldScriptTests.java | 4 +- .../ExpressionNumberSortScriptTests.java | 2 +- .../ExpressionTermsSetQueryTests.java | 2 +- .../painless/NeedsScoreTests.java | 11 +- .../index/mapper/RankFeatureFieldMapper.java | 4 +- .../index/mapper/RankFeaturesFieldMapper.java | 4 +- .../index/mapper/ScaledFloatFieldMapper.java | 4 +- .../mapper/ScaledFloatFieldTypeTests.java | 8 +- .../join/mapper/MetaJoinFieldMapper.java | 4 +- .../join/mapper/ParentIdFieldMapper.java | 4 +- .../join/mapper/ParentJoinFieldMapper.java | 6 +- .../percolator/PercolateQueryBuilder.java | 3 +- .../ICUCollationKeywordFieldMapper.java | 4 +- .../mapper/murmur3/Murmur3FieldMapper.java | 4 +- .../org/elasticsearch/index/IndexService.java | 4 +- .../elasticsearch/index/IndexSortConfig.java | 2 +- .../org/elasticsearch/index/IndexWarmer.java | 6 +- .../fielddata/IndexFieldDataService.java | 16 +- .../index/mapper/BinaryFieldMapper.java | 4 +- .../index/mapper/BooleanFieldMapper.java | 4 +- .../index/mapper/DateFieldMapper.java | 4 +- .../index/mapper/GeoPointFieldMapper.java | 4 +- .../index/mapper/IdFieldMapper.java | 4 +- .../index/mapper/IndexFieldMapper.java | 5 +- .../index/mapper/IpFieldMapper.java | 4 +- .../index/mapper/KeywordFieldMapper.java | 4 +- .../index/mapper/MappedFieldType.java | 12 +- .../index/mapper/NumberFieldMapper.java | 4 +- .../index/mapper/RangeFieldMapper.java | 4 +- .../index/mapper/SeqNoFieldMapper.java | 4 +- .../index/mapper/TextFieldMapper.java | 4 +- .../index/mapper/TypeFieldMapper.java | 4 +- .../index/query/QueryShardContext.java | 22 +- .../search/lookup/DocLookup.java | 1 - .../search/lookup/SearchLookup.java | 84 ++++++- .../index/IndexSortSettingsTests.java | 35 +-- .../fielddata/IndexFieldDataServiceTests.java | 61 ++++- .../index/mapper/IdFieldMapperTests.java | 9 +- .../index/mapper/NumberFieldTypeTests.java | 4 +- .../index/mapper/TextFieldMapperTests.java | 8 +- .../index/mapper/TypeFieldMapperTests.java | 7 +- .../index/query/QueryShardContextTests.java | 233 +++++++++++++++++- .../index/shard/IndexShardTests.java | 4 +- .../search/sort/AbstractSortTestCase.java | 9 +- .../aggregations/AggregatorTestCase.java | 23 +- .../mapper/HistogramFieldMapper.java | 5 +- .../mapper/ConstantKeywordFieldMapper.java | 4 +- .../mapper/FlatObjectFieldMapper.java | 6 +- .../mapper/FlatObjectFieldLookupTests.java | 7 +- .../mapper/FlatObjectIndexFieldDataTests.java | 8 +- .../GeoShapeWithDocValuesFieldMapper.java | 4 +- .../mapper/DenseVectorFieldMapper.java | 4 +- .../mapper/SparseVectorFieldMapper.java | 4 +- .../wildcard/mapper/WildcardFieldMapper.java | 3 +- .../mapper/WildcardFieldMapperTests.java | 10 +- 55 files changed, 567 insertions(+), 145 deletions(-) diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java index 02c5f46f2ea..f178479f21d 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java @@ -19,8 +19,8 @@ package org.elasticsearch.script.expression; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -64,7 +64,7 @@ public class ExpressionFieldScriptTests extends ESTestCase { when(fieldData.load(anyObject())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, ignored -> fieldData, null); + lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData, null); } private FieldScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java index 7d59a9b4a45..f24c6ab08ca 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java @@ -63,7 +63,7 @@ public class ExpressionNumberSortScriptTests extends ESTestCase { when(fieldData.load(anyObject())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, ignored -> fieldData, null); + lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData, null); } private NumberSortScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java index 820c7ab584a..3744576b5b1 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java @@ -63,7 +63,7 @@ public class ExpressionTermsSetQueryTests extends ESTestCase { when(fieldData.load(anyObject())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, ignored -> fieldData, null); + lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData, null); } private TermsSetQueryScript.LeafFactory compile(String expression) { diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index eeb636d6697..e9a6ca60509 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.script.NumberSortScript; import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import java.util.Collections; @@ -47,23 +46,21 @@ public class NeedsScoreTests extends ESSingleNodeTestCase { PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts); QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null); - SearchLookup lookup = new SearchLookup(index.mapperService(), shardContext::getForField, null); NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap()); - NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), lookup); + NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); assertFalse(ss.needs_score()); factory = service.compile(null, "doc['d'].value", NumberSortScript.CONTEXT, Collections.emptyMap()); - ss = factory.newFactory(Collections.emptyMap(), lookup); + ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); assertFalse(ss.needs_score()); factory = service.compile(null, "1/_score", NumberSortScript.CONTEXT, Collections.emptyMap()); - ss = factory.newFactory(Collections.emptyMap(), lookup); + ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); assertTrue(ss.needs_score()); factory = service.compile(null, "doc['d'].value * _score", NumberSortScript.CONTEXT, Collections.emptyMap()); - ss = factory.newFactory(Collections.emptyMap(), lookup); + ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); assertTrue(ss.needs_score()); } - } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index a33df960db4..3513406d2ae 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -31,11 +31,13 @@ import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * A {@link FieldMapper} that exposes Lucene's {@link FeatureField}. @@ -118,7 +120,7 @@ public class RankFeatureFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { throw new IllegalArgumentException("[rank_feature] fields do not support sorting, scripting or aggregating"); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index a088c6f70f7..59b8ccb7f05 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -27,10 +27,12 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * A {@link FieldMapper} that exposes Lucene's {@link FeatureField} as a sparse @@ -91,7 +93,7 @@ public class RankFeaturesFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { throw new IllegalArgumentException("[rank_features] fields do not support sorting, scripting or aggregating"); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index e23deca7a88..656656a5265 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.math.BigDecimal; @@ -57,6 +58,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** A {@link FieldMapper} for scaled floats. Values are internally multiplied * by a scaling factor and rounded to the closest long. */ @@ -216,7 +218,7 @@ public class ScaledFloatFieldMapper extends ParametrizedFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return (cache, breakerService, mapperService) -> { final IndexNumericFieldData scaledValues = new SortedNumericIndexFieldData.Builder( diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java index 64d7115196c..e893198a0cc 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java @@ -150,7 +150,9 @@ public class ScaledFloatFieldTypeTests extends FieldTypeTestCase { // single-valued ScaledFloatFieldMapper.ScaledFloatFieldType f1 = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float1", scalingFactor); - IndexNumericFieldData fielddata = (IndexNumericFieldData) f1.fielddataBuilder("index").build(null, null, null); + IndexNumericFieldData fielddata = (IndexNumericFieldData) f1.fielddataBuilder("index", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, null); assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE); LeafNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0)); SortedNumericDoubleValues values = leafFieldData.getDoubleValues(); @@ -161,7 +163,9 @@ public class ScaledFloatFieldTypeTests extends FieldTypeTestCase { // multi-valued ScaledFloatFieldMapper.ScaledFloatFieldType f2 = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float2", scalingFactor); - fielddata = (IndexNumericFieldData) f2.fielddataBuilder("index").build(null, null, null); + fielddata = (IndexNumericFieldData) f2.fielddataBuilder("index", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, null); leafFieldData = fielddata.load(reader.leaves().get(0)); values = leafFieldData.getDoubleValues(); assertTrue(values.advanceExact(0)); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 590214dacdf..161c61137d8 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -33,10 +33,12 @@ import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.function.Supplier; /** * Simple field mapper hack to ensure that there is a one and only {@link ParentJoinFieldMapper} per mapping. @@ -90,7 +92,7 @@ public class MetaJoinFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 497139ca081..b9bf5ade734 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -43,12 +43,14 @@ import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; /** * A field mapper used internally by the {@link ParentJoinFieldMapper} to index @@ -108,7 +110,7 @@ public final class ParentIdFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 058fefa9b51..aef2495932e 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -34,13 +34,13 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; @@ -48,6 +48,7 @@ import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -58,6 +59,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; /** * A {@link FieldMapper} that creates hierarchical joins (parent-join) between documents in the same index. @@ -218,7 +220,7 @@ public final class ParentJoinFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index adafa935f14..76bbf1930fa 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -725,7 +725,8 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder> IFD getForField(MappedFieldType fieldType) { - IndexFieldData.Builder builder = fieldType.fielddataBuilder(shardContext.getFullyQualifiedIndex().getName()); + IndexFieldData.Builder builder = fieldType.fielddataBuilder(shardContext.getFullyQualifiedIndex().getName(), + shardContext::lookup); IndexFieldDataCache cache = new IndexFieldDataCache.None(); CircuitBreakerService circuitBreaker = new NoneCircuitBreakerService(); return (IFD) builder.build(cache, circuitBreaker, shardContext.getMapperService()); diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index e3c05781a0e..50d19c3c668 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -46,6 +46,7 @@ import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; @@ -54,6 +55,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; public class ICUCollationKeywordFieldMapper extends FieldMapper { @@ -105,7 +107,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 8156d5f9efa..117b36366e2 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -42,10 +42,12 @@ import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class Murmur3FieldMapper extends FieldMapper { @@ -105,7 +107,7 @@ public class Murmur3FieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 5aa382f883c..15ddf74eae3 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -200,7 +200,9 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust // The sort order is validated right after the merge of the mapping later in the process. this.indexSortSupplier = () -> indexSettings.getIndexSortConfig().buildIndexSort( mapperService::fieldType, - indexFieldData::getForField + fieldType -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), () -> { + throw new UnsupportedOperationException("search lookup not available for index sorting"); + }) ); } else { this.indexSortSupplier = () -> null; diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 68b7f504ed7..a0f65f0ee69 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -202,7 +202,7 @@ public final class IndexSortConfig { try { fieldData = fieldDataLookup.apply(ft); } catch (Exception e) { - throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]"); + throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]", e); } if (fieldData == null) { throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]"); diff --git a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java index 1bf3f071e70..d873f99c704 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java +++ b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java @@ -131,7 +131,11 @@ public final class IndexWarmer { executor.execute(() -> { try { final long start = System.nanoTime(); - IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType); + IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType, + indexFieldDataService.index().getName(), + () -> { + throw new UnsupportedOperationException("search lookup not available when warming an index"); + }); IndexFieldData global = ifd.loadGlobal(reader); if (reader.leaves().isEmpty() == false) { global.load(reader.leaves().get(0)); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 324728e8b09..a0bf9b2082c 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.Closeable; import java.io.IOException; @@ -38,6 +39,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class IndexFieldDataService extends AbstractIndexComponent implements Closeable { public static final String FIELDDATA_CACHE_VALUE_NODE = "node"; @@ -106,14 +108,16 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Clo ExceptionsHelper.maybeThrowRuntimeAndSuppress(exceptions); } - public > IFD getForField(MappedFieldType fieldType) { - return getForField(fieldType, index().getName()); - } - + /** + * Returns fielddata for the provided field type, given the provided fully qualified index name, while also making + * a {@link SearchLookup} supplier available that is required for runtime fields. + */ @SuppressWarnings("unchecked") - public > IFD getForField(MappedFieldType fieldType, String fullyQualifiedIndexName) { + public > IFD getForField(MappedFieldType fieldType, + String fullyQualifiedIndexName, + Supplier searchLookup) { final String fieldName = fieldType.name(); - IndexFieldData.Builder builder = fieldType.fielddataBuilder(fullyQualifiedIndexName); + IndexFieldData.Builder builder = fieldType.fielddataBuilder(fullyQualifiedIndexName, searchLookup); IndexFieldDataCache cache; synchronized (this) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 7861a35f8ac..8cc0d1fe556 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; @@ -46,6 +47,7 @@ import java.util.Base64; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class BinaryFieldMapper extends ParametrizedFieldMapper { @@ -124,7 +126,7 @@ public class BinaryFieldMapper extends ParametrizedFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new BytesBinaryIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 2f0a397e71d..c3e25fdde3b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -39,6 +39,7 @@ import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; @@ -46,6 +47,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * A field mapper for boolean fields. @@ -170,7 +172,7 @@ public class BooleanFieldMapper extends ParametrizedFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.BOOLEAN); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index ae8ada087ac..66bf3f8d138 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -50,6 +50,7 @@ import org.elasticsearch.index.query.DateRangeIncludingNowQuery; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.DateTimeException; @@ -64,6 +65,7 @@ import java.util.Locale; import java.util.Map; import java.util.function.Function; import java.util.function.LongSupplier; +import java.util.function.Supplier; import static org.elasticsearch.common.time.DateUtils.toLong; @@ -446,7 +448,7 @@ public final class DateFieldMapper extends ParametrizedFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), resolution.numericType()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 0b5092ca370..fa6b510ccd0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -38,12 +38,14 @@ import org.elasticsearch.index.mapper.GeoPointFieldMapper.ParsedGeoPoint; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.VectorGeoPointShapeQueryProcessor; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * Field Mapper for geo_point types. @@ -184,7 +186,7 @@ public class GeoPointFieldMapper extends AbstractPointGeometryFieldMapper searchLookup) { failIfNoDocValues(); return new AbstractLatLonPointIndexFieldData.Builder(name(), CoreValuesSourceType.GEOPOINT); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index b241403ac39..806538ad121 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -46,6 +46,7 @@ import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; @@ -53,6 +54,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.Supplier; /** * A mapper for the _id field. It does nothing since _id is neither indexed nor @@ -140,7 +142,7 @@ public class IdFieldMapper extends MetadataFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { final IndexFieldData.Builder fieldDataBuilder = new PagedBytesIndexFieldData.Builder( name(), TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 6b28bebe6d4..20cb3c8153e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -25,10 +25,11 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; - +import java.util.function.Supplier; public class IndexFieldMapper extends MetadataFieldMapper { @@ -62,7 +63,7 @@ public class IndexFieldMapper extends MetadataFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { return new ConstantIndexFieldData.Builder(mapperService -> fullyQualifiedIndexName, name(), CoreValuesSourceType.BYTES); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index db85f3d6a51..dc4b6148179 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.net.InetAddress; @@ -48,6 +49,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** A {@link FieldMapper} for ip addresses. */ public class IpFieldMapper extends ParametrizedFieldMapper { @@ -268,7 +270,7 @@ public class IpFieldMapper extends ParametrizedFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), IpScriptDocValues::new, CoreValuesSourceType.IP); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 88de254d10f..7610bbbce7a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.UncheckedIOException; @@ -48,6 +49,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; /** * A field mapper for keywords. This mapper accepts strings and indexes them as-is. @@ -246,7 +248,7 @@ public final class KeywordFieldMapper extends ParametrizedFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index dd538cdb012..cd1e229c4aa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; @@ -55,6 +56,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.function.Function; +import java.util.function.Supplier; /** * This defines the core properties and functions to operate on a field. @@ -83,12 +85,12 @@ public abstract class MappedFieldType { * Return a fielddata builder for this field * * @param fullyQualifiedIndexName the name of the index this field-data is build for - * + * @param searchLookup a {@link SearchLookup} supplier to allow for accessing other fields values in the context of runtime fields * @throws IllegalArgumentException if the fielddata is not supported on this type. * An IllegalArgumentException is needed in order to return an http error 400 * when this error occurs in a request. see: {@link org.elasticsearch.ExceptionsHelper#status} */ - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { throw new IllegalArgumentException("Fielddata is not supported on field [" + name() + "] of type [" + typeName() + "]"); } @@ -154,7 +156,9 @@ public abstract class MappedFieldType { */ public boolean isAggregatable() { try { - fielddataBuilder(""); + fielddataBuilder("", () -> { + throw new UnsupportedOperationException("SearchLookup not available"); + }); return true; } catch (IllegalArgumentException e) { return false; @@ -214,7 +218,7 @@ public abstract class MappedFieldType { + "] which is of type [" + typeName() + "]"); } - public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxDeterminizedStates, + public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) { throw new QueryShardException(context, "Can only use regexp queries on keyword and text fields - not on [" + name + "] which is of type [" + typeName() + "]"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 0246fa29eda..3970bdcbc36 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -53,6 +53,7 @@ import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; @@ -63,6 +64,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.function.Function; +import java.util.function.Supplier; /** A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. */ public class NumberFieldMapper extends ParametrizedFieldMapper { @@ -927,7 +929,7 @@ public class NumberFieldMapper extends ParametrizedFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), type.numericType()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 82f9bad89f9..6f08e30dca9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -46,6 +46,7 @@ import org.elasticsearch.index.fielddata.plain.BinaryIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.net.InetAddress; @@ -61,6 +62,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Supplier; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD; @@ -221,7 +223,7 @@ public class RangeFieldMapper extends FieldMapper { public RangeType rangeType() { return rangeType; } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new BinaryIndexFieldData.Builder(name(), CoreValuesSourceType.RANGE); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 343c9a84d25..4d2317e7e89 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -34,11 +34,13 @@ import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.function.Supplier; /** * Mapper for the {@code _seq_no} field. @@ -168,7 +170,7 @@ public class SeqNoFieldMapper extends MetadataFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 00e8cce59fc..899fbde2bc2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -72,6 +72,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -82,6 +83,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.function.IntPredicate; +import java.util.function.Supplier; import static org.elasticsearch.index.mapper.TypeParsers.checkNull; import static org.elasticsearch.index.mapper.TypeParsers.parseTextField; @@ -760,7 +762,7 @@ public class TextFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { if (fielddata == false) { throw new IllegalArgumentException("Text fields are not optimised for operations that require per-document " + "field data like aggregations and sorting, so these operations are disabled by default. Please use a " diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 3d2c85b4db6..e213639c327 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -44,6 +44,7 @@ import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.support.QueryParsers; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Arrays; @@ -52,6 +53,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Function; +import java.util.function.Supplier; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; @@ -90,7 +92,7 @@ public class TypeFieldMapper extends MetadataFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { Function typeFunction = mapperService -> mapperService.documentMapper().type(); return new ConstantIndexFieldData.Builder(typeFunction, name(), CoreValuesSourceType.BYTES); } diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 43b0b2099dd..a6915ebb8f3 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -32,6 +32,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.search.Queries; @@ -72,10 +73,10 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; -import java.util.function.BiFunction; import java.util.function.BooleanSupplier; import java.util.function.LongSupplier; import java.util.function.Predicate; +import java.util.function.Supplier; import static java.util.Collections.unmodifiableMap; @@ -93,7 +94,7 @@ public class QueryShardContext extends QueryRewriteContext { private final MapperService mapperService; private final SimilarityService similarityService; private final BitsetFilterCache bitsetFilterCache; - private final BiFunction> indexFieldDataService; + private final TriFunction, IndexFieldData> indexFieldDataService; private final int shardId; private final IndexSearcher searcher; private String[] types = Strings.EMPTY_ARRAY; @@ -122,7 +123,7 @@ public class QueryShardContext extends QueryRewriteContext { IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, - BiFunction> indexFieldDataLookup, + TriFunction, IndexFieldData> indexFieldDataLookup, MapperService mapperService, SimilarityService similarityService, ScriptService scriptService, @@ -152,7 +153,7 @@ public class QueryShardContext extends QueryRewriteContext { IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, - BiFunction> indexFieldDataLookup, + TriFunction, IndexFieldData> indexFieldDataLookup, MapperService mapperService, SimilarityService similarityService, ScriptService scriptService, @@ -223,7 +224,8 @@ public class QueryShardContext extends QueryRewriteContext { } public > IFD getForField(MappedFieldType fieldType) { - return (IFD) indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName()); + return (IFD) indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), + () -> this.lookup().forkAndTrackFieldReferences(fieldType.name())); } public void addNamedQuery(String name, Query query) { @@ -324,11 +326,13 @@ public class QueryShardContext extends QueryRewriteContext { private SearchLookup lookup = null; public SearchLookup lookup() { - if (lookup == null) { - lookup = new SearchLookup(getMapperService(), - mappedFieldType -> indexFieldDataService.apply(mappedFieldType, fullyQualifiedIndex.getName()), types); + if (this.lookup == null) { + this.lookup = new SearchLookup( + getMapperService(), + (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup), + types); } - return lookup; + return this.lookup; } public NestedScope nestedScope() { diff --git a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java index 9cfe121fe7e..c2a40f1d1f8 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java @@ -30,7 +30,6 @@ public class DocLookup { private final MapperService mapperService; private final Function> fieldDataLookup; - @Nullable private final String[] types; diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index 8f4b5143dc6..4de8ef8fc13 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -25,21 +25,89 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import java.util.function.Function; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Supplier; public class SearchLookup { + /** + * The maximum depth of field dependencies. + * When a runtime field's doc values depends on another runtime field's doc values, + * which depends on another runtime field's doc values and so on, it can + * make a very deep stack, which we want to limit. + */ + private static final int MAX_FIELD_CHAIN_DEPTH = 5; - final DocLookup docMap; + /** + * The chain of fields for which this lookup was created, used for detecting + * loops caused by runtime fields referring to other runtime fields. The chain is empty + * for the "top level" lookup created for the entire search. When a lookup is used to load + * fielddata for a field, we fork it and make sure the field name name isn't in the chain, + * then add it to the end. So the lookup for the a field named {@code a} will be {@code ["a"]}. If + * that field looks up the values of a field named {@code b} then + * {@code b}'s chain will contain {@code ["a", "b"]}. + */ + private final Set fieldChain; + private final DocLookup docMap; + private final SourceLookup sourceLookup; + private final FieldsLookup fieldsLookup; + private final BiFunction, IndexFieldData> fieldDataLookup; - final SourceLookup sourceLookup; - - final FieldsLookup fieldsLookup; - - public SearchLookup(MapperService mapperService, Function> fieldDataLookup, + /** + * Create the top level field lookup for a search request. Provides a way to look up fields from doc_values, + * stored fields, or _source. + */ + public SearchLookup(MapperService mapperService, + BiFunction, IndexFieldData> fieldDataLookup, @Nullable String[] types) { - docMap = new DocLookup(mapperService, fieldDataLookup, types); + this.fieldChain = Collections.emptySet(); + docMap = new DocLookup(mapperService, + fieldType -> fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name())), + types); sourceLookup = new SourceLookup(); fieldsLookup = new FieldsLookup(mapperService, types); + this.fieldDataLookup = fieldDataLookup; + } + + /** + * Create a new {@link SearchLookup} that looks fields up the same as the one provided as argument, + * while also tracking field references starting from the provided field name. It detects cycles + * and prevents resolving fields that depend on more than {@link #MAX_FIELD_CHAIN_DEPTH} fields. + * @param searchLookup the existing lookup to create a new one from + * @param fieldChain the chain of fields that required the field currently being loaded + */ + private SearchLookup(SearchLookup searchLookup, Set fieldChain) { + this.fieldChain = Collections.unmodifiableSet(fieldChain); + this.docMap = new DocLookup(searchLookup.docMap.mapperService(), + fieldType -> searchLookup.fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name())), + searchLookup.docMap.getTypes()); + this.sourceLookup = searchLookup.sourceLookup; + this.fieldsLookup = searchLookup.fieldsLookup; + this.fieldDataLookup = searchLookup.fieldDataLookup; + } + + /** + * Creates a copy of the current {@link SearchLookup} that looks fields up in the same way, but also tracks field references + * in order to detect cycles and prevent resolving fields that depend on more than {@link #MAX_FIELD_CHAIN_DEPTH} other fields. + * @param field the field being referred to, for which fielddata needs to be loaded + * @return the new lookup + * @throws IllegalArgumentException if a cycle is detected in the fields required to build doc values, or if the field + * being resolved depends on more than {@link #MAX_FIELD_CHAIN_DEPTH} + */ + public final SearchLookup forkAndTrackFieldReferences(String field) { + Objects.requireNonNull(field, "field cannot be null"); + Set newFieldChain = new LinkedHashSet<>(fieldChain); + if (newFieldChain.add(field) == false) { + String message = String.join(" -> ", newFieldChain) + " -> " + field; + throw new IllegalArgumentException("Cyclic dependency detected while resolving runtime fields: " + message); + } + if (newFieldChain.size() > MAX_FIELD_CHAIN_DEPTH) { + throw new IllegalArgumentException("Field requires resolving too many dependent fields: " + String.join(" -> ", newFieldChain)); + } + return new SearchLookup(this, newFieldChain); } public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) { diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 5eed34cfe72..ffc08f9e1d6 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -19,15 +19,11 @@ package org.elasticsearch.index; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; -import java.io.IOException; - import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; import static org.hamcrest.Matchers.containsString; @@ -35,28 +31,15 @@ import static org.hamcrest.Matchers.equalTo; public class IndexSortSettingsTests extends ESTestCase { private static IndexSettings indexSettings(Settings settings) { - return indexSettings(settings, null); + return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); } - private static IndexSettings indexSettings(Settings settings, Version version) { - final Settings newSettings; - if (version != null) { - newSettings = Settings.builder() - .put(settings) - .put(IndexMetadata.SETTING_VERSION_CREATED, version) - .build(); - } else { - newSettings = settings; - } - return new IndexSettings(newIndexMeta("test", newSettings), Settings.EMPTY); - } - - public void testNoIndexSort() throws IOException { + public void testNoIndexSort() { IndexSettings indexSettings = indexSettings(EMPTY_SETTINGS); assertFalse(indexSettings.getIndexSortConfig().hasIndexSort()); } - public void testSimpleIndexSort() throws IOException { + public void testSimpleIndexSort() { Settings settings = Settings.builder() .put("index.sort.field", "field1") .put("index.sort.order", "asc") @@ -74,7 +57,7 @@ public class IndexSortSettingsTests extends ESTestCase { assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MAX)); } - public void testIndexSortWithArrays() throws IOException { + public void testIndexSortWithArrays() { Settings settings = Settings.builder() .putList("index.sort.field", "field1", "field2") .putList("index.sort.order", "asc", "desc") @@ -95,7 +78,7 @@ public class IndexSortSettingsTests extends ESTestCase { assertNull(config.sortSpecs[1].mode); } - public void testInvalidIndexSort() throws IOException { + public void testInvalidIndexSort() { final Settings settings = Settings.builder() .put("index.sort.field", "field1") .put("index.sort.order", "asc, desc") @@ -105,7 +88,7 @@ public class IndexSortSettingsTests extends ESTestCase { assertThat(exc.getMessage(), containsString("index.sort.field:[field1] index.sort.order:[asc, desc], size mismatch")); } - public void testInvalidIndexSortWithArray() throws IOException { + public void testInvalidIndexSortWithArray() { final Settings settings = Settings.builder() .put("index.sort.field", "field1") .putList("index.sort.order", new String[] {"asc", "desc"}) @@ -116,7 +99,7 @@ public class IndexSortSettingsTests extends ESTestCase { containsString("index.sort.field:[field1] index.sort.order:[asc, desc], size mismatch")); } - public void testInvalidOrder() throws IOException { + public void testInvalidOrder() { final Settings settings = Settings.builder() .put("index.sort.field", "field1") .put("index.sort.order", "invalid") @@ -126,7 +109,7 @@ public class IndexSortSettingsTests extends ESTestCase { assertThat(exc.getMessage(), containsString("Illegal sort order:invalid")); } - public void testInvalidMode() throws IOException { + public void testInvalidMode() { final Settings settings = Settings.builder() .put("index.sort.field", "field1") .put("index.sort.mode", "invalid") @@ -136,7 +119,7 @@ public class IndexSortSettingsTests extends ESTestCase { assertThat(exc.getMessage(), containsString("Illegal sort mode: invalid")); } - public void testInvalidMissing() throws IOException { + public void testInvalidMissing() { final Settings settings = Settings.builder() .put("index.sort.field", "field1") .put("index.sort.missing", "default") diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 25d3a1f1c77..cebdf5f4be1 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; @@ -46,18 +47,23 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.mockito.Matchers; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import static org.hamcrest.Matchers.containsString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { @@ -74,7 +80,9 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1)); final MappedFieldType stringMapper = new KeywordFieldMapper.Builder("string").build(ctx).fieldType(); ifdService.clear(); - IndexFieldData fd = ifdService.getForField(stringMapper); + IndexFieldData fd = ifdService.getForField(stringMapper, "test", () -> { + throw new UnsupportedOperationException(); + }); assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData); for (MappedFieldType mapper : Arrays.asList( @@ -84,23 +92,47 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { new NumberFieldMapper.Builder("long", NumberFieldMapper.NumberType.LONG, false, true).build(ctx).fieldType() )) { ifdService.clear(); - fd = ifdService.getForField(mapper); + fd = ifdService.getForField(mapper, "test", () -> { + throw new UnsupportedOperationException(); + }); assertTrue(fd instanceof SortedNumericIndexFieldData); } final MappedFieldType floatMapper = new NumberFieldMapper.Builder("float", NumberFieldMapper.NumberType.FLOAT, false, true) .build(ctx).fieldType(); ifdService.clear(); - fd = ifdService.getForField(floatMapper); + fd = ifdService.getForField(floatMapper, "test", () -> { + throw new UnsupportedOperationException(); + }); assertTrue(fd instanceof SortedNumericIndexFieldData); final MappedFieldType doubleMapper = new NumberFieldMapper.Builder("double", NumberFieldMapper.NumberType.DOUBLE, false, true) .build(ctx).fieldType(); ifdService.clear(); - fd = ifdService.getForField(doubleMapper); + fd = ifdService.getForField(doubleMapper, "test", () -> { + throw new UnsupportedOperationException(); + }); assertTrue(fd instanceof SortedNumericIndexFieldData); } + public void testGetForFieldRuntimeField() { + final IndexService indexService = createIndex("test"); + final IndicesService indicesService = getInstanceFromNode(IndicesService.class); + final IndexFieldDataService ifdService = new IndexFieldDataService(indexService.getIndexSettings(), + indicesService.getIndicesFieldDataCache(), indicesService.getCircuitBreakerService(), indexService.mapperService()); + final SetOnce> searchLookupSetOnce = new SetOnce<>(); + MappedFieldType ft = mock(MappedFieldType.class); + when(ft.fielddataBuilder(Matchers.any(), Matchers.any())).thenAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + Supplier searchLookup = (Supplier)invocationOnMock.getArguments()[1]; + searchLookupSetOnce.set(searchLookup); + return (IndexFieldData.Builder) (cache, breakerService, mapperService) -> null; + }); + SearchLookup searchLookup = new SearchLookup(null, null, null); + ifdService.getForField(ft, "qualified", () -> searchLookup); + assertSame(searchLookup, searchLookupSetOnce.get().get()); + } + public void testClearField() throws Exception { final IndexService indexService = createIndex("test"); final IndicesService indicesService = getInstanceFromNode(IndicesService.class); @@ -130,8 +162,12 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { onRemovalCalled.incrementAndGet(); } }); - IndexFieldData ifd1 = ifdService.getForField(mapper1); - IndexFieldData ifd2 = ifdService.getForField(mapper2); + IndexFieldData ifd1 = ifdService.getForField(mapper1, "test", () -> { + throw new UnsupportedOperationException(); + }); + IndexFieldData ifd2 = ifdService.getForField(mapper2, "test", () -> { + throw new UnsupportedOperationException(); + }); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData loadField1 = ifd1.load(leafReaderContext); LeafFieldData loadField2 = ifd2.load(leafReaderContext); @@ -200,7 +236,9 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { onRemovalCalled.incrementAndGet(); } }); - IndexFieldData ifd = ifdService.getForField(mapper1); + IndexFieldData ifd = ifdService.getForField(mapper1, "test", () -> { + throw new UnsupportedOperationException(); + }); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData load = ifd.load(leafReaderContext); assertEquals(1, onCacheCalled.get()); @@ -256,10 +294,14 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { IndexFieldDataService ifds = new IndexFieldDataService(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), cache, null, null); if (ft.hasDocValues()) { - ifds.getForField(ft); // no exception + ifds.getForField(ft, "test", () -> { + throw new UnsupportedOperationException(); + }); // no exception } else { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ifds.getForField(ft)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ifds.getForField(ft, "test", () -> { + throw new UnsupportedOperationException(); + })); assertThat(e.getMessage(), containsString("doc values")); } } finally { @@ -283,5 +325,4 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field")); doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field", true, false, Collections.emptyMap())); } - } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java index c47b8144602..29716293492 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java @@ -82,7 +82,9 @@ public class IdFieldMapperTests extends ESSingleNodeTestCase { mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE); IdFieldMapper.IdFieldType ft = (IdFieldMapper.IdFieldType) service.mapperService().fieldType("_id"); - ft.fielddataBuilder("test").build(null, null, mapperService); + ft.fielddataBuilder("test", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, mapperService); assertWarnings(ID_FIELD_DATA_DEPRECATION_MESSAGE); client().admin().cluster().prepareUpdateSettings() @@ -90,7 +92,9 @@ public class IdFieldMapperTests extends ESSingleNodeTestCase { .get(); try { IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, - () -> ft.fielddataBuilder("test").build(null, null, mapperService)); + () -> ft.fielddataBuilder("test", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, mapperService)); assertThat(exc.getMessage(), containsString(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())); } finally { // unset cluster setting @@ -98,7 +102,6 @@ public class IdFieldMapperTests extends ESSingleNodeTestCase { .setTransientSettings(Settings.builder().putNull(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())) .get(); } - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index b770bb93da6..274c764478e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -451,7 +451,9 @@ public class NumberFieldTypeTests extends FieldTypeTestCase { // Create an index writer configured with the same index sort. NumberFieldType fieldType = new NumberFieldType("field", type); - IndexNumericFieldData fielddata = (IndexNumericFieldData) fieldType.fielddataBuilder("index").build(null, null, null); + IndexNumericFieldData fielddata = (IndexNumericFieldData) fieldType.fielddataBuilder("index", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, null); SortField sortField = fielddata.sortField(null, MultiValueMode.MIN, null, randomBoolean()); IndexWriterConfig writerConfig = new IndexWriterConfig(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 117f8aed816..1de3e50fc08 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -422,12 +422,16 @@ public class TextFieldMapperTests extends FieldMapperTestCase2 disabledMapper.fieldType("field").fielddataBuilder("test") + () -> disabledMapper.fieldType("field").fielddataBuilder("test", () -> { + throw new UnsupportedOperationException(); + }) ); assertThat(e.getMessage(), containsString("Text fields are not optimised for operations that require per-document field data")); MapperService enabledMapper = createMapperService(fieldMapping(b -> b.field("type", "text").field("fielddata", true))); - enabledMapper.fieldType("field").fielddataBuilder("test"); // no exception this time + enabledMapper.fieldType("field").fielddataBuilder("test", () -> { + throw new UnsupportedOperationException(); + }); // no exception this time e = expectThrows( MapperParsingException.class, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java index b89cc0c6e98..57ae10f7b4a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java @@ -30,9 +30,9 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.plugins.Plugin; @@ -68,8 +68,9 @@ public class TypeFieldMapperTests extends ESSingleNodeTestCase { w.close(); MappedFieldType ft = mapperService.fieldType(TypeFieldMapper.NAME); - IndexOrdinalsFieldData fd = (IndexOrdinalsFieldData) ft.fielddataBuilder("test") - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService(), mapperService); + IndexOrdinalsFieldData fd = (IndexOrdinalsFieldData) ft.fielddataBuilder("test", () -> { + throw new UnsupportedOperationException(); + }).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService(), mapperService); LeafOrdinalsFieldData afd = fd.load(r.leaves().get(0)); SortedSetDocValues values = afd.getOrdinalsValues(); assertTrue(values.advanceExact(0)); diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 93fe7e6e079..ff9e6ef3fd0 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -18,9 +18,24 @@ */ package org.elasticsearch.index.query; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -29,21 +44,32 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.search.lookup.LeafDocLookup; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.function.BiFunction; +import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -153,7 +179,109 @@ public class QueryShardContextTests extends ESTestCase { assertFalse(context.indexSortedOnField("non_sort_field")); } + public void testFielddataLookupSelfReference() { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + //simulate a runtime field that depends on itself e.g. field: doc['field'] + return leafLookup.doc().get(field).toString(); + }); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("field", queryShardContext)); + assertEquals("Cyclic dependency detected while resolving runtime fields: field -> field", iae.getMessage()); + } + + public void testFielddataLookupLooseLoop() { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + //simulate a runtime field cycle: 1: doc['2'] 2: doc['3'] 3: doc['4'] 4: doc['1'] + if (field.equals("4")) { + return leafLookup.doc().get("1").toString(); + } + return leafLookup.doc().get(Integer.toString(Integer.parseInt(field) + 1)).toString(); + }); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); + assertEquals("Cyclic dependency detected while resolving runtime fields: 1 -> 2 -> 3 -> 4 -> 1", iae.getMessage()); + } + + public void testFielddataLookupTerminatesInLoop() { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + //simulate a runtime field cycle: 1: doc['2'] 2: doc['3'] 3: doc['4'] 4: doc['4'] + if (field.equals("4")) { + return leafLookup.doc().get("4").toString(); + } + return leafLookup.doc().get(Integer.toString(Integer.parseInt(field) + 1)).toString(); + }); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); + assertEquals("Cyclic dependency detected while resolving runtime fields: 1 -> 2 -> 3 -> 4 -> 4", iae.getMessage()); + } + + public void testFielddataLookupSometimesLoop() throws IOException { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + if (docId == 0) { + return field + "_" + docId; + } else { + assert docId == 1; + if (field.equals("field4")) { + return leafLookup.doc().get("field1").toString(); + } + int i = Integer.parseInt(field.substring(field.length() - 1)); + return leafLookup.doc().get("field" + (i + 1)).toString(); + } + }); + List values = collect("field1", queryShardContext, new TermQuery(new Term("indexed_field", "first"))); + assertEquals(Collections.singletonList("field1_0"), values); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("field1", queryShardContext)); + assertEquals("Cyclic dependency detected while resolving runtime fields: field1 -> field2 -> field3 -> field4 -> field1", + iae.getMessage()); + } + + public void testFielddataLookupBeyondMaxDepth() { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + int i = Integer.parseInt(field); + return leafLookup.doc().get(Integer.toString(i + 1)).toString(); + }); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); + assertEquals("Field requires resolving too many dependent fields: 1 -> 2 -> 3 -> 4 -> 5 -> 6", iae.getMessage()); + } + + public void testFielddataLookupReferencesBelowMaxDepth() throws IOException { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + int i = Integer.parseInt(field.substring(field.length() - 1)); + if (i == 5) { + return "test"; + } else { + ScriptDocValues scriptDocValues = leafLookup.doc().get("field" + (i + 1)); + return scriptDocValues.get(0).toString() + docId; + } + }); + assertEquals(Arrays.asList("test0000", "test1111"), collect("field1", queryShardContext)); + } + + public void testFielddataLookupOneFieldManyReferences() throws IOException { + int numFields = randomIntBetween(5, 20); + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + if (field.equals("field")) { + StringBuilder value = new StringBuilder(); + for (int i = 0; i < numFields; i++) { + value.append(leafLookup.doc().get("field" + i).get(0)); + } + return value.toString(); + } else { + return "test" + docId; + } + }); + StringBuilder expectedFirstDoc = new StringBuilder(); + StringBuilder expectedSecondDoc = new StringBuilder(); + for (int i = 0; i < numFields; i++) { + expectedFirstDoc.append("test0"); + expectedSecondDoc.append("test1"); + } + assertEquals(Arrays.asList(expectedFirstDoc.toString(), expectedSecondDoc.toString()), collect("field", queryShardContext)); + } + public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) { + return createQueryShardContext(indexUuid, clusterAlias, null); + } + + private static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias, + TriFunction runtimeDocValues) { IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder("index"); indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT) .put("index.number_of_shards", 1) @@ -165,12 +293,113 @@ public class QueryShardContextTests extends ESTestCase { MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.index()).thenReturn(indexMetadata.getIndex()); + if (runtimeDocValues != null) { + when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { + String fieldName = (String)fieldTypeInv.getArguments()[0]; + return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); + }); + } final long nowInMillis = randomNonNegativeLong(); - return new QueryShardContext( 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, - (mappedFieldType, idxName) -> mappedFieldType.fielddataBuilder(idxName).build(null, null, null), + (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null, null), mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); } + + private static MappedFieldType mockFieldType(String fieldName, BiFunction runtimeDocValues) { + MappedFieldType fieldType = mock(MappedFieldType.class); + when(fieldType.name()).thenReturn(fieldName); + when(fieldType.fielddataBuilder(any(), any())).thenAnswer(builderInv -> { + @SuppressWarnings("unchecked") + Supplier searchLookup = ((Supplier) builderInv.getArguments()[1]); + IndexFieldData indexFieldData = mock(IndexFieldData.class); + when(indexFieldData.load(any())).thenAnswer(loadArgs -> { + LeafReaderContext leafReaderContext = (LeafReaderContext) loadArgs.getArguments()[0]; + LeafFieldData leafFieldData = mock(LeafFieldData.class); + when(leafFieldData.getScriptValues()).thenAnswer(scriptValuesArgs -> new ScriptDocValues() { + String value; + + @Override + public int size() { + return 1; + } + + @Override + public String get(int index) { + assert index == 0; + return value; + } + + @Override + public void setNextDocId(int docId) { + assert docId >= 0; + LeafSearchLookup leafLookup = searchLookup.get().getLeafSearchLookup(leafReaderContext); + leafLookup.setDocument(docId); + value = runtimeDocValues.apply(leafLookup, docId); + } + }); + return leafFieldData; + }); + IndexFieldData.Builder builder = mock(IndexFieldData.Builder.class); + when(builder.build(any(), any(), any())).thenAnswer(buildInv -> indexFieldData); + return builder; + }); + return fieldType; + } + + private static List collect(String field, QueryShardContext queryShardContext) throws IOException { + return collect(field, queryShardContext, new MatchAllDocsQuery()); + } + + private static List collect(String field, QueryShardContext queryShardContext, Query query) throws IOException { + List result = new ArrayList<>(); + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexWriter.addDocument(Collections.singletonList(new StringField("indexed_field", "first", Field.Store.NO))); + indexWriter.addDocument(Collections.singletonList(new StringField("indexed_field", "second", Field.Store.NO))); + try (DirectoryReader reader = indexWriter.getReader()) { + IndexSearcher searcher = newSearcher(reader); + MappedFieldType fieldType = queryShardContext.fieldMapper(field); + IndexFieldData indexFieldData; + if (randomBoolean()) { + indexFieldData = queryShardContext.getForField(fieldType); + } else { + indexFieldData = queryShardContext.lookup().doc().getForField(fieldType); + } + searcher.search(query, new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) { + ScriptDocValues scriptValues = indexFieldData.load(context).getScriptValues(); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) {} + + @Override + public void collect(int doc) throws IOException { + ScriptDocValues scriptDocValues; + if(randomBoolean()) { + LeafDocLookup leafDocLookup = queryShardContext.lookup().doc().getLeafDocLookup(context); + leafDocLookup.setDocument(doc); + scriptDocValues = leafDocLookup.get(field); + } else { + scriptDocValues = scriptValues; + } + scriptDocValues.setNextDocId(doc); + for (int i = 0; i < scriptDocValues.size(); i++) { + result.add(scriptDocValues.get(i).toString()); + } + } + }; + } + }); + } + return result; + } + } + } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 9bae3d688be..a3c7126c911 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2467,7 +2467,9 @@ public class IndexShardTests extends IndexShardTestCase { new IndexFieldDataCache.Listener() {}); IndexFieldDataService indexFieldDataService = new IndexFieldDataService(shard.indexSettings, indicesFieldDataCache, new NoneCircuitBreakerService(), shard.mapperService()); - IndexFieldData.Global ifd = indexFieldDataService.getForField(foo); + IndexFieldData.Global ifd = indexFieldDataService.getForField(foo, "test", () -> { + throw new UnsupportedOperationException("search lookup not available"); + }); FieldDataStats before = shard.fieldData().stats("foo"); assertThat(before.getMemorySizeInBytes(), equalTo(0L)); FieldDataStats after = null; diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 888c677e1dc..a153c6dcfc2 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -23,6 +23,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SortField; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; @@ -56,6 +57,7 @@ import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.junit.AfterClass; @@ -65,8 +67,8 @@ import org.mockito.Mockito; import java.io.IOException; import java.util.Collections; import java.util.Map; -import java.util.function.BiFunction; import java.util.function.Function; +import java.util.function.Supplier; import static java.util.Collections.emptyList; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; @@ -194,8 +196,9 @@ public abstract class AbstractSortTestCase> extends EST IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build()); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class)); - BiFunction> indexFieldDataLookup = (fieldType, fieldIndexName) -> { - IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName); + TriFunction, IndexFieldData> indexFieldDataLookup = + (fieldType, fieldIndexName, searchLookup) -> { + IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null, null); }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 5f8c910da63..680542b0883 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -49,6 +49,7 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lease.Releasable; @@ -118,6 +119,7 @@ import org.elasticsearch.search.fetch.subphase.FetchDocValuesPhase; import org.elasticsearch.search.fetch.subphase.FetchSourcePhase; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalAggregationTestCase; import org.junit.After; @@ -132,9 +134,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import static java.util.Collections.emptyMap; @@ -318,8 +320,11 @@ public abstract class AggregatorTestCase extends ESTestCase { new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() { }), circuitBreakerService, mapperService); when(searchContext.getForField(Mockito.any(MappedFieldType.class))) - .thenAnswer(invocationOnMock -> ifds.getForField((MappedFieldType) invocationOnMock.getArguments()[0])); - + .thenAnswer(invocationOnMock -> ifds.getForField((MappedFieldType) invocationOnMock.getArguments()[0], + indexSettings.getIndex().getName(), + () -> { + throw new UnsupportedOperationException("search lookup not available"); + })); QueryShardContext queryShardContext = queryShardContextMock(contextIndexSearcher, mapperService, indexSettings, circuitBreakerService, bigArrays); when(searchContext.getQueryShardContext()).thenReturn(queryShardContext); @@ -386,11 +391,11 @@ public abstract class AggregatorTestCase extends ESTestCase { /** * Sub-tests that need a more complex index field data provider can override this */ - protected BiFunction> getIndexFieldDataLookup(MapperService mapperService, - CircuitBreakerService circuitBreakerService) { - return (fieldType, s) -> fieldType.fielddataBuilder(mapperService.getIndexSettings().getIndex().getName()) + protected TriFunction, IndexFieldData> getIndexFieldDataLookup( + MapperService mapperService, CircuitBreakerService circuitBreakerService) { + return (fieldType, s, searchLookup) -> fieldType.fielddataBuilder( + mapperService.getIndexSettings().getIndex().getName(), searchLookup) .build(new IndexFieldDataCache.None(), circuitBreakerService, mapperService); - } /** @@ -712,7 +717,9 @@ public abstract class AggregatorTestCase extends ESTestCase { } private ValuesSourceType fieldToVST(MappedFieldType fieldType) { - return fieldType.fielddataBuilder("").build(null, null, null).getValuesSourceType(); + return fieldType.fielddataBuilder("", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, null).getValuesSourceType(); } /** diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index c9f8bf64df7..3930a7c8818 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.analytics.mapper; import com.carrotsearch.hppc.DoubleArrayList; import com.carrotsearch.hppc.IntArrayList; - import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; @@ -53,6 +52,7 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; @@ -61,6 +61,7 @@ import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -191,7 +192,7 @@ public class HistogramFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new IndexFieldData.Builder() { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 6096977cfac..e4831bcd8e6 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; @@ -47,6 +48,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; /** * A {@link FieldMapper} that assigns every document the same value. @@ -140,7 +142,7 @@ public class ConstantKeywordFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { return new ConstantIndexFieldData.Builder(mapperService -> value, name(), CoreValuesSourceType.BYTES); } diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 203bde6e37b..c4283ac49a1 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -51,6 +51,7 @@ import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; @@ -59,6 +60,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import static org.elasticsearch.index.mapper.TypeParsers.parseField; @@ -319,7 +321,7 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new KeyedFlatObjectFieldData.Builder(name(), key, CoreValuesSourceType.BYTES); } @@ -478,7 +480,7 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java index 03e7c119218..14fe135553b 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.lookup.SearchLookup; @@ -23,7 +23,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.function.Function; +import java.util.function.BiFunction; +import java.util.function.Supplier; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; @@ -161,7 +162,7 @@ public class FlatObjectFieldLookupTests extends ESTestCase { = new KeyedFlatObjectFieldType( "field", true, true, "key2", false, Collections.emptyMap()); when(mapperService.fieldType("json.key2")).thenReturn(fieldType2); - Function> fieldDataSupplier = fieldType -> { + BiFunction, IndexFieldData> fieldDataSupplier = (fieldType, searchLookup) -> { KeyedFlatObjectFieldType keyedFieldType = (KeyedFlatObjectFieldType) fieldType; return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; }; diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java index e7e1b0578fc..917830355e4 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java @@ -80,7 +80,9 @@ public class FlatObjectIndexFieldDataTests extends ESSingleNodeTestCase { // Load global field data for subfield 'key'. KeyedFlatObjectFieldType fieldType1 = fieldMapper.keyedFieldType("key"); - IndexFieldData ifd1 = ifdService.getForField(fieldType1); + IndexFieldData ifd1 = ifdService.getForField(fieldType1, "test", () -> { + throw new UnsupportedOperationException("search lookup not available"); + }); assertTrue(ifd1 instanceof KeyedFlatObjectFieldData); KeyedFlatObjectFieldData fieldData1 = (KeyedFlatObjectFieldData) ifd1; @@ -90,7 +92,9 @@ public class FlatObjectIndexFieldDataTests extends ESSingleNodeTestCase { // Load global field data for the subfield 'other_key'. KeyedFlatObjectFieldType fieldType2 = fieldMapper.keyedFieldType("other_key"); - IndexFieldData ifd2 = ifdService.getForField(fieldType2); + IndexFieldData ifd2 = ifdService.getForField(fieldType2, "test", () -> { + throw new UnsupportedOperationException("search lookup not available"); + }); assertTrue(ifd2 instanceof KeyedFlatObjectFieldData); KeyedFlatObjectFieldData fieldData2 = (KeyedFlatObjectFieldData) ifd2; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 34750b89890..adfa2914528 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xpack.spatial.index.fielddata.AbstractLatLonShapeIndexFieldData; import org.elasticsearch.xpack.spatial.index.fielddata.CentroidCalculator; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; @@ -37,6 +38,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * Extension of {@link org.elasticsearch.index.mapper.GeoShapeFieldMapper} that supports docValues @@ -131,7 +133,7 @@ public class GeoShapeWithDocValuesFieldMapper extends GeoShapeFieldMapper { super(name, indexed, hasDocValues, meta); } - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new AbstractLatLonShapeIndexFieldData.Builder(name(), GeoShapeValuesSourceType.instance()); } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 28e83be9648..1d147d6a4f5 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xpack.vectors.query.VectorIndexFieldData; import java.io.IOException; @@ -38,6 +39,7 @@ import java.nio.ByteBuffer; import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -128,7 +130,7 @@ public class DenseVectorFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { return new VectorIndexFieldData.Builder(name(), true, CoreValuesSourceType.BYTES); } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 5b6e2ee9ddb..81a01838cd8 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -30,12 +30,14 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xpack.vectors.query.VectorIndexFieldData; import java.io.IOException; import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -108,7 +110,7 @@ public class SparseVectorFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { return new VectorIndexFieldData.Builder(name(), false, CoreValuesSourceType.BYTES); } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index a8e0c42d05e..1cbe7230d7c 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -66,6 +66,7 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -861,7 +862,7 @@ public class WildcardFieldMapper extends FieldMapper { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new IndexFieldData.Builder() { @Override diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 5c1bc799948..98f0e3377f3 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -38,10 +38,10 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.ByteRunAutomaton; -//import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.collect.List; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; @@ -57,6 +57,7 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -67,7 +68,7 @@ import org.mockito.Mockito; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; -import java.util.function.BiFunction; +import java.util.function.Supplier; import static org.elasticsearch.index.mapper.FieldMapperTestCase.fetchSourceValue; import static org.hamcrest.Matchers.equalTo; @@ -812,8 +813,9 @@ public class WildcardFieldMapperTests extends ESTestCase { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build()); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class)); - BiFunction> indexFieldDataLookup = (fieldType, fieldIndexName) -> { - IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName); + TriFunction, IndexFieldData> indexFieldDataLookup = + (fieldType, fieldIndexName, searchLookup) -> { + IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null, null); }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup,