From 5a0b6d1977a44872c6018070d7a5ccf0d04f6c9d Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 21 Nov 2017 15:14:03 +0100 Subject: [PATCH] Use the primary_term field to identify parent documents (#27469) This change stops indexing the `_primary_term` field for nested documents to allow fast retrieval of parent documents. Today we create a docvalues field for children to ensure we have a dense datastructure on disk. Yet, since we only use the primary term to tie-break on when we see the same seqID on indexing having a dense datastructure is less important. We can use this now to improve the nested docs performance and it's memory footprint. Relates to #24362 --- .../common/lucene/search/Queries.java | 16 ++++++++-- .../index/cache/bitset/BitsetFilterCache.java | 2 +- .../index/mapper/SeqNoFieldMapper.java | 11 +++++-- .../index/mapper/TypeFieldMapper.java | 2 +- .../index/query/NestedQueryBuilder.java | 4 +-- .../index/shard/ShardSplittingQuery.java | 6 ++-- .../search/DefaultSearchContext.java | 2 +- .../bucket/nested/NestedAggregator.java | 4 ++- .../nested/ReverseNestedAggregator.java | 2 +- .../search/fetch/FetchPhase.java | 6 ++-- .../search/sort/SortBuilder.java | 2 +- .../apache/lucene/search/QueriesTests.java | 19 ++++++++++-- .../index/mapper/TypeFieldTypeTests.java | 7 +++-- .../index/shard/ShardSplittingQueryTests.java | 29 ++++++++++++++----- .../search/DefaultSearchContextTests.java | 1 + .../bucket/nested/NestedAggregatorTests.java | 16 +++++++++- .../nested/ReverseNestedAggregatorTests.java | 2 ++ .../PercolatorMatchedSlotSubFetchPhase.java | 7 ++++- 18 files changed, 105 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java index 5129cd5485e..31184bb4351 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java @@ -25,13 +25,16 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; +import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.TypeFieldMapper; import java.util.List; @@ -62,12 +65,19 @@ public class Queries { return new PrefixQuery(new Term(TypeFieldMapper.NAME, new BytesRef("__"))); } - public static Query newNonNestedFilter() { - // TODO: this is slow, make it a positive query - return new BooleanQuery.Builder() + /** + * Creates a new non-nested docs query + * @param indexVersionCreated the index version created since newer indices can identify a parent field more efficiently + */ + public static Query newNonNestedFilter(Version indexVersionCreated) { + if (indexVersionCreated.onOrAfter(Version.V_7_0_0_alpha1)) { + return new DocValuesFieldExistsQuery(SeqNoFieldMapper.PRIMARY_TERM_NAME); + } else { + return new BooleanQuery.Builder() .add(new MatchAllDocsQuery(), Occur.FILTER) .add(newNestedFilter(), Occur.MUST_NOT) .build(); + } } public static BooleanQuery filtered(@Nullable Query query, @Nullable Query filter) { diff --git a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java index 2de8f01dc71..6af9c5eeb6e 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java +++ b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java @@ -249,7 +249,7 @@ public final class BitsetFilterCache extends AbstractIndexComponent implements I } if (hasNested) { - warmUp.add(Queries.newNonNestedFilter()); + warmUp.add(Queries.newNonNestedFilter(indexSettings.getIndexVersionCreated())); } final CountDownLatch latch = new CountDownLatch(searcher.reader().leaves().size() * warmUp.size()); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 55fc1333d20..391705d719c 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -252,11 +253,17 @@ public class SeqNoFieldMapper extends MetadataFieldMapper { // we share the parent docs fields to ensure good compression SequenceIDFields seqID = context.seqID(); assert seqID != null; - for (int i = 1; i < context.docs().size(); i++) { + int numDocs = context.docs().size(); + final Version versionCreated = context.mapperService().getIndexSettings().getIndexVersionCreated(); + final boolean includePrimaryTerm = versionCreated.before(Version.V_7_0_0_alpha1); + for (int i = 1; i < numDocs; i++) { final Document doc = context.docs().get(i); doc.add(seqID.seqNo); doc.add(seqID.seqNoDocValue); - doc.add(seqID.primaryTerm); + if (includePrimaryTerm) { + // primary terms are used to distinguish between parent and nested docs since 6.1.0 + doc.add(seqID.primaryTerm); + } } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index d0e30e77c9e..712e9edec9e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -156,7 +156,7 @@ public class TypeFieldMapper extends MetadataFieldMapper { .anyMatch(indexType::equals)) { if (context.getMapperService().hasNested()) { // type filters are expected not to match nested docs - return Queries.newNonNestedFilter(); + return Queries.newNonNestedFilter(context.indexVersionCreated()); } else { return new MatchAllDocsQuery(); } diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java index 4e3429e1a20..ccf5ad71078 100644 --- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java @@ -282,7 +282,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder Query innerQuery; ObjectMapper objectMapper = context.nestedScope().getObjectMapper(); if (objectMapper == null) { - parentFilter = context.bitsetFilter(Queries.newNonNestedFilter()); + parentFilter = context.bitsetFilter(Queries.newNonNestedFilter(context.indexVersionCreated())); } else { parentFilter = context.bitsetFilter(objectMapper.nestedTypeFilter()); } @@ -377,7 +377,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder SearchHit hit = hits[i]; Query rawParentFilter; if (parentObjectMapper == null) { - rawParentFilter = Queries.newNonNestedFilter(); + rawParentFilter = Queries.newNonNestedFilter(context.indexShard().indexSettings().getIndexVersionCreated()); } else { rawParentFilter = parentObjectMapper.nestedTypeFilter(); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java index da13d4ba15f..0919630abcc 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java @@ -71,7 +71,7 @@ final class ShardSplittingQuery extends Query { } this.indexMetaData = indexMetaData; this.shardId = shardId; - this.nestedParentBitSetProducer = hasNested ? newParentDocBitSetProducer() : null; + this.nestedParentBitSetProducer = hasNested ? newParentDocBitSetProducer(indexMetaData.getCreationVersion()) : null; } @Override public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) { @@ -336,9 +336,9 @@ final class ShardSplittingQuery extends Query { * than once. There is no point in using BitsetFilterCache#BitSetProducerWarmer since we use this only as a delete by query which is * executed on a recovery-private index writer. There is no point in caching it and it won't have a cache hit either. */ - private static BitSetProducer newParentDocBitSetProducer() { + private static BitSetProducer newParentDocBitSetProducer(Version indexVersionCreated) { return context -> { - Query query = Queries.newNonNestedFilter(); + Query query = Queries.newNonNestedFilter(indexVersionCreated); final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(context); final IndexSearcher searcher = new IndexSearcher(topLevelContext); searcher.setQueryCache(null); diff --git a/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 34c3c03f758..1356a1458a2 100644 --- a/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -270,7 +270,7 @@ final class DefaultSearchContext extends SearchContext { && typeFilter == null // when a _type filter is set, it will automatically exclude nested docs && new NestedHelper(mapperService()).mightMatchNestedDocs(query) && (aliasFilter == null || new NestedHelper(mapperService()).mightMatchNestedDocs(aliasFilter))) { - filters.add(Queries.newNonNestedFilter()); + filters.add(Queries.newNonNestedFilter(mapperService().getIndexSettings().getIndexVersionCreated())); } if (aliasFilter != null) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java index c7d721baa67..da0793be59d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java @@ -62,7 +62,9 @@ class NestedAggregator extends BucketsAggregator implements SingleBucketAggregat List pipelineAggregators, Map metaData, boolean collectsFromSingleBucket) throws IOException { super(name, factories, context, parentAggregator, pipelineAggregators, metaData); - Query parentFilter = parentObjectMapper != null ? parentObjectMapper.nestedTypeFilter() : Queries.newNonNestedFilter(); + + Query parentFilter = parentObjectMapper != null ? parentObjectMapper.nestedTypeFilter() + : Queries.newNonNestedFilter(context.mapperService().getIndexSettings().getIndexVersionCreated()); this.parentFilter = context.bitsetFilterCache().getBitSetProducer(parentFilter); this.childFilter = childObjectMapper.nestedTypeFilter(); this.collectsFromSingleBucket = collectsFromSingleBucket; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java index cf45a09ef61..338093af377 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java @@ -54,7 +54,7 @@ public class ReverseNestedAggregator extends BucketsAggregator implements Single throws IOException { super(name, factories, context, parent, pipelineAggregators, metaData); if (objectMapper == null) { - parentFilter = Queries.newNonNestedFilter(); + parentFilter = Queries.newNonNestedFilter(context.mapperService().getIndexSettings().getIndexVersionCreated()); } else { parentFilter = objectMapper.nestedTypeFilter(); } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 4596162970b..683f7bbde16 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -181,7 +181,9 @@ public class FetchPhase implements SearchPhase { private int findRootDocumentIfNested(SearchContext context, LeafReaderContext subReaderContext, int subDocId) throws IOException { if (context.mapperService().hasNested()) { - BitSet bits = context.bitsetFilterCache().getBitSetProducer(Queries.newNonNestedFilter()).getBitSet(subReaderContext); + BitSet bits = context.bitsetFilterCache() + .getBitSetProducer(Queries.newNonNestedFilter(context.indexShard().indexSettings().getIndexVersionCreated())) + .getBitSet(subReaderContext); if (!bits.get(subDocId)) { return bits.nextSetBit(subDocId); } @@ -345,7 +347,7 @@ public class FetchPhase implements SearchPhase { } parentFilter = nestedParentObjectMapper.nestedTypeFilter(); } else { - parentFilter = Queries.newNonNestedFilter(); + parentFilter = Queries.newNonNestedFilter(context.indexShard().indexSettings().getIndexVersionCreated()); } Query childFilter = nestedObjectMapper.nestedTypeFilter(); diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java index 61130fc34d6..9537e288919 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java @@ -212,7 +212,7 @@ public abstract class SortBuilder> implements NamedWrit Query parentQuery; ObjectMapper objectMapper = context.nestedScope().getObjectMapper(); if (objectMapper == null) { - parentQuery = Queries.newNonNestedFilter(); + parentQuery = Queries.newNonNestedFilter(context.indexVersionCreated()); } else { parentQuery = objectMapper.nestedTypeFilter(); } diff --git a/core/src/test/java/org/apache/lucene/search/QueriesTests.java b/core/src/test/java/org/apache/lucene/search/QueriesTests.java index 43723ddcdbb..d117b741a2e 100644 --- a/core/src/test/java/org/apache/lucene/search/QueriesTests.java +++ b/core/src/test/java/org/apache/lucene/search/QueriesTests.java @@ -19,15 +19,28 @@ package org.apache.lucene.search; +import org.elasticsearch.Version; import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; public class QueriesTests extends ESTestCase { public void testNonNestedQuery() { - // This is a custom query that extends AutomatonQuery and want to make sure the equals method works - assertEquals(Queries.newNonNestedFilter(), Queries.newNonNestedFilter()); - assertEquals(Queries.newNonNestedFilter().hashCode(), Queries.newNonNestedFilter().hashCode()); + for (Version version : VersionUtils.allVersions()) { + // This is a custom query that extends AutomatonQuery and want to make sure the equals method works + assertEquals(Queries.newNonNestedFilter(version), Queries.newNonNestedFilter(version)); + assertEquals(Queries.newNonNestedFilter(version).hashCode(), Queries.newNonNestedFilter(version).hashCode()); + if (version.onOrAfter(Version.V_7_0_0_alpha1)) { + assertEquals(Queries.newNonNestedFilter(version), new DocValuesFieldExistsQuery(SeqNoFieldMapper.PRIMARY_TERM_NAME)); + } else { + assertEquals(Queries.newNonNestedFilter(version), new BooleanQuery.Builder() + .add(new MatchAllDocsQuery(), BooleanClause.Occur.FILTER) + .add(Queries.newNestedFilter(), BooleanClause.Occur.MUST_NOT) + .build()); + } + } } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java index 8f64e051929..9cb6cf61796 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/TypeFieldTypeTests.java @@ -44,6 +44,7 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.test.VersionUtils; import org.mockito.Mockito; import java.io.IOException; @@ -58,14 +59,16 @@ public class TypeFieldTypeTests extends FieldTypeTestCase { public void testTermsQueryWhenTypesAreDisabled() throws Exception { QueryShardContext context = Mockito.mock(QueryShardContext.class); + Version indexVersionCreated = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.CURRENT); Settings indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build(); IndexMetaData indexMetaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(indexSettings).build(); IndexSettings mockSettings = new IndexSettings(indexMetaData, Settings.EMPTY); Mockito.when(context.getIndexSettings()).thenReturn(mockSettings); + Mockito.when(context.indexVersionCreated()).thenReturn(indexVersionCreated); MapperService mapperService = Mockito.mock(MapperService.class); Set types = Collections.emptySet(); @@ -84,7 +87,7 @@ public class TypeFieldTypeTests extends FieldTypeTestCase { Mockito.when(mapperService.hasNested()).thenReturn(true); query = ft.termQuery("my_type", context); - assertEquals(Queries.newNonNestedFilter(), query); + assertEquals(Queries.newNonNestedFilter(context.indexVersionCreated()), query); types = Collections.singleton("other_type"); Mockito.when(mapperService.types()).thenReturn(types); diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java index c0b492b0cb6..9dcb712a05d 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.test.ESTestCase; @@ -51,6 +52,7 @@ import java.util.List; public class ShardSplittingQueryTests extends ESTestCase { public void testSplitOnID() throws IOException { + SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); Directory dir = newFSDirectory(createTempDir()); final int numDocs = randomIntBetween(50, 100); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); @@ -76,13 +78,15 @@ public class ShardSplittingQueryTests extends ESTestCase { } docs.add(Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), - new SortedNumericDocValuesField("shard_id", shardId) + new SortedNumericDocValuesField("shard_id", shardId), + sequenceIDFields.primaryTerm )); writer.addDocuments(docs); } else { writer.addDocument(Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), - new SortedNumericDocValuesField("shard_id", shardId) + new SortedNumericDocValuesField("shard_id", shardId), + sequenceIDFields.primaryTerm )); } } @@ -95,6 +99,7 @@ public class ShardSplittingQueryTests extends ESTestCase { } public void testSplitOnRouting() throws IOException { + SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); Directory dir = newFSDirectory(createTempDir()); final int numDocs = randomIntBetween(50, 100); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); @@ -122,14 +127,16 @@ public class ShardSplittingQueryTests extends ESTestCase { docs.add(Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), - new SortedNumericDocValuesField("shard_id", shardId) + new SortedNumericDocValuesField("shard_id", shardId), + sequenceIDFields.primaryTerm )); writer.addDocuments(docs); } else { writer.addDocument(Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), - new SortedNumericDocValuesField("shard_id", shardId) + new SortedNumericDocValuesField("shard_id", shardId), + sequenceIDFields.primaryTerm )); } } @@ -140,6 +147,7 @@ public class ShardSplittingQueryTests extends ESTestCase { } public void testSplitOnIdOrRouting() throws IOException { + SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); Directory dir = newFSDirectory(createTempDir()); final int numDocs = randomIntBetween(50, 100); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); @@ -160,13 +168,15 @@ public class ShardSplittingQueryTests extends ESTestCase { rootDoc = Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), - new SortedNumericDocValuesField("shard_id", shardId) + new SortedNumericDocValuesField("shard_id", shardId), + sequenceIDFields.primaryTerm ); } else { shardId = OperationRouting.generateShardId(metaData, Integer.toString(j), null); rootDoc = Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), - new SortedNumericDocValuesField("shard_id", shardId) + new SortedNumericDocValuesField("shard_id", shardId), + sequenceIDFields.primaryTerm ); } @@ -194,6 +204,7 @@ public class ShardSplittingQueryTests extends ESTestCase { public void testSplitOnRoutingPartitioned() throws IOException { + SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); Directory dir = newFSDirectory(createTempDir()); final int numDocs = randomIntBetween(50, 100); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); @@ -223,14 +234,16 @@ public class ShardSplittingQueryTests extends ESTestCase { docs.add(Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), - new SortedNumericDocValuesField("shard_id", shardId) + new SortedNumericDocValuesField("shard_id", shardId), + sequenceIDFields.primaryTerm )); writer.addDocuments(docs); } else { writer.addDocument(Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), - new SortedNumericDocValuesField("shard_id", shardId) + new SortedNumericDocValuesField("shard_id", shardId), + sequenceIDFields.primaryTerm )); } } diff --git a/core/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/core/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index c20724b8a92..539bdef17d3 100644 --- a/core/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/core/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -102,6 +102,7 @@ public class DefaultSearchContextTests extends ESTestCase { IndexMetaData indexMetaData = IndexMetaData.builder("index").settings(settings).build(); IndexSettings indexSettings = new IndexSettings(indexMetaData, Settings.EMPTY); when(indexService.getIndexSettings()).thenReturn(indexSettings); + when(mapperService.getIndexSettings()).thenReturn(indexSettings); BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService()); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index f6e18f82804..d3d72fe152b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -36,11 +36,13 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.mapper.UidFieldMapper; import org.elasticsearch.search.aggregations.AggregatorTestCase; @@ -56,6 +58,7 @@ import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.sum.InternalSum; import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.util.ArrayList; @@ -73,6 +76,9 @@ public class NestedAggregatorTests extends AggregatorTestCase { private static final String MAX_AGG_NAME = "maxAgg"; private static final String SUM_AGG_NAME = "sumAgg"; + private final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + + public void testNoDocs() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { @@ -120,6 +126,7 @@ public class NestedAggregatorTests extends AggregatorTestCase { UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); + document.add(sequenceIDFields.primaryTerm); documents.add(document); iw.addDocuments(documents); } @@ -168,6 +175,7 @@ public class NestedAggregatorTests extends AggregatorTestCase { UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); + document.add(sequenceIDFields.primaryTerm); documents.add(document); iw.addDocuments(documents); } @@ -216,6 +224,7 @@ public class NestedAggregatorTests extends AggregatorTestCase { UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); + document.add(sequenceIDFields.primaryTerm); documents.add(document); iw.addDocuments(documents); } @@ -254,6 +263,7 @@ public class NestedAggregatorTests extends AggregatorTestCase { public void testResetRootDocId() throws Exception { IndexWriterConfig iwc = new IndexWriterConfig(null); iwc.setMergePolicy(NoMergePolicy.INSTANCE); + SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, iwc)) { List documents = new ArrayList<>(); @@ -274,6 +284,7 @@ public class NestedAggregatorTests extends AggregatorTestCase { document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); + document.add(sequenceIDFields.primaryTerm); documents.add(document); iw.addDocuments(documents); iw.commit(); @@ -288,6 +299,7 @@ public class NestedAggregatorTests extends AggregatorTestCase { document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#2", UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); + document.add(sequenceIDFields.primaryTerm); documents.add(document); iw.addDocuments(documents); documents.clear(); @@ -299,6 +311,7 @@ public class NestedAggregatorTests extends AggregatorTestCase { document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#3", UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); + document.add(sequenceIDFields.primaryTerm); documents.add(document); iw.addDocuments(documents); @@ -314,7 +327,7 @@ public class NestedAggregatorTests extends AggregatorTestCase { fieldType.setName(VALUE_FIELD_NAME); BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.add(Queries.newNonNestedFilter(), BooleanClause.Occur.MUST); + bq.add(Queries.newNonNestedFilter(VersionUtils.randomVersion(random())), BooleanClause.Occur.MUST); bq.add(new TermQuery(new Term(UidFieldMapper.NAME, "type#2")), BooleanClause.Occur.MUST_NOT); Nested nested = search(newSearcher(indexReader, false, true), @@ -550,6 +563,7 @@ public class NestedAggregatorTests extends AggregatorTestCase { Document document = new Document(); document.add(new Field(UidFieldMapper.NAME, "book#" + id, UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "book", TypeFieldMapper.Defaults.FIELD_TYPE)); + document.add(sequenceIDFields.primaryTerm); for (String author : authors) { document.add(new SortedSetDocValuesField("author", new BytesRef(author))); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java index 70e6c355940..7281228f3c3 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.mapper.UidFieldMapper; import org.elasticsearch.search.aggregations.AggregatorTestCase; @@ -108,6 +109,7 @@ public class ReverseNestedAggregatorTests extends AggregatorTestCase { TypeFieldMapper.Defaults.FIELD_TYPE)); long value = randomNonNegativeLong() % 10000; document.add(new SortedNumericDocValuesField(VALUE_FIELD_NAME, value)); + document.add(SeqNoFieldMapper.SequenceIDFields.emptySeqID().primaryTerm); if (numNestedDocs > 0) { expectedMaxValue = Math.max(expectedMaxValue, value); expectedParentDocs++; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java index 163c4183dd4..5f4bcc35a46 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.Weight; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; +import org.elasticsearch.Version; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.search.SearchHit; @@ -65,7 +66,11 @@ final class PercolatorMatchedSlotSubFetchPhase implements FetchSubPhase { for (PercolateQuery percolateQuery : percolateQueries) { String fieldName = singlePercolateQuery ? FIELD_NAME_PREFIX : FIELD_NAME_PREFIX + "_" + percolateQuery.getName(); IndexSearcher percolatorIndexSearcher = percolateQuery.getPercolatorIndexSearcher(); - Weight weight = percolatorIndexSearcher.createNormalizedWeight(Queries.newNonNestedFilter(), false); + // there is a bug in lucene's MemoryIndex that doesn't allow us to use docValues here... + // See https://issues.apache.org/jira/browse/LUCENE-8055 + // for now we just use version 6.0 version to find nested parent + final Version version = Version.V_6_0_0; //context.mapperService().getIndexSettings().getIndexVersionCreated(); + Weight weight = percolatorIndexSearcher.createNormalizedWeight(Queries.newNonNestedFilter(version), false); Scorer s = weight.scorer(percolatorIndexSearcher.getIndexReader().leaves().get(0)); int memoryIndexMaxDoc = percolatorIndexSearcher.getIndexReader().maxDoc(); BitSet rootDocs = BitSet.of(s.iterator(), memoryIndexMaxDoc);