From d241c4898e5d5ab87284cc3b351989d26947b552 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Fri, 19 May 2017 17:11:23 +0200 Subject: [PATCH] Removes parent child fielddata specialization (#24737) This change removes the field data specialization needed for the parent field and replaces it with a simple DocValuesIndexFieldData. The underlying global ordinals are retrieved via a new function called IndexOrdinalsFieldData#getOrdinalMap. The children aggregation is also modified to use a simple WithOrdinals value source rather than the deleted WithOrdinals.Parent. Relates #20257 --- .../fielddata/AtomicParentChildFieldData.java | 43 -- .../fielddata/IndexOrdinalsFieldData.java | 7 +- .../fielddata/IndexParentChildFieldData.java | 46 -- .../index/fielddata/UidIndexFieldData.java | 6 + .../ordinals/GlobalOrdinalsBuilder.java | 4 +- .../GlobalOrdinalsIndexFieldData.java | 75 +++- .../InternalGlobalOrdinalsIndexFieldData.java | 99 ----- .../AbstractAtomicParentChildFieldData.java | 128 ------ .../plain/AbstractIndexOrdinalsFieldData.java | 6 + .../plain/ParentChildIndexFieldData.java | 401 ------------------ .../SortedSetDVOrdinalsIndexFieldData.java | 6 + .../index/mapper/ParentFieldMapper.java | 4 +- .../aggregations/support/ValuesSource.java | 38 -- .../support/ValuesSourceConfig.java | 5 +- .../AbstractStringFieldDataTestCase.java | 2 + .../fielddata/ParentChildFieldDataTests.java | 278 ------------ .../ChildrenAggregationBuilder.java | 33 +- .../ChildrenAggregatorFactory.java | 22 +- .../ParentToChildrenAggregator.java | 23 +- .../join/query/HasChildQueryBuilder.java | 27 +- .../join/query/HasParentQueryBuilder.java | 23 +- .../ParentToChildrenAggregatorTests.java | 3 +- .../join/query/ChildQuerySearchIT.java | 4 +- 23 files changed, 171 insertions(+), 1112 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java delete mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/IndexParentChildFieldData.java delete mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java delete mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java delete mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java delete mode 100644 core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java deleted file mode 100644 index f88d7c58774..00000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata; - -import org.apache.lucene.index.SortedDocValues; - -import java.util.Set; - -/** - * Specialization of {@link AtomicFieldData} for parent/child mappings. - */ -public interface AtomicParentChildFieldData extends AtomicFieldData { - - /** - * Return the set of types there is a mapping for. - */ - Set types(); - - /** - * Return the mapping for the given type. The returned - * {@link SortedDocValues} will map doc IDs to the identifier of their - * parent. - */ - SortedDocValues getOrdinalsValues(String type); - -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java index cb1471179c2..2e714fc80a1 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java @@ -21,7 +21,7 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; - +import org.apache.lucene.index.MultiDocValues; /** @@ -42,4 +42,9 @@ public interface IndexOrdinalsFieldData extends IndexFieldData.Global { - - /** - * Load a global view of the ordinals for the given {@link IndexReader}, - * potentially from a cache. - */ - @Override - IndexParentChildFieldData loadGlobal(DirectoryReader indexReader); - - /** - * Load a global view of the ordinals for the given {@link IndexReader}. - */ - @Override - IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception; - -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java index e8dea836e32..d0e8285be2e 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BytesRef; @@ -99,6 +100,11 @@ public final class UidIndexFieldData implements IndexOrdinalsFieldData { return new UidIndexFieldData(index, type, idFieldData.localGlobalDirect(indexReader)); } + @Override + public MultiDocValues.OrdinalMap getOrdinalMap() { + return idFieldData.getOrdinalMap(); + } + static final class UidAtomicFieldData implements AtomicOrdinalsFieldData { private final BytesRef prefix; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java index 49140968ca7..0a8ab3ccd4b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java @@ -74,7 +74,7 @@ public enum GlobalOrdinalsBuilder { new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS) ); } - return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), + return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), atomicFD, ordinalMap, memorySizeInBytes, scriptFunction ); } @@ -108,7 +108,7 @@ public enum GlobalOrdinalsBuilder { subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); - return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), + return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), atomicFD, ordinalMap, 0, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION ); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java index 2055208021e..795e4b99202 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java @@ -20,6 +20,8 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.SortField; import org.apache.lucene.util.Accountable; import org.elasticsearch.common.Nullable; @@ -29,23 +31,39 @@ import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData; import org.elasticsearch.search.MultiValueMode; import java.util.Collection; import java.util.Collections; +import java.util.function.Function; /** * {@link IndexFieldData} base class for concrete global ordinals implementations. */ -public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable { +public class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable { private final String fieldName; private final long memorySizeInBytes; - protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, long memorySizeInBytes) { + private final MultiDocValues.OrdinalMap ordinalMap; + private final Atomic[] atomicReaders; + private final Function> scriptFunction; + + + protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd, + MultiDocValues.OrdinalMap ordinalMap, long memorySizeInBytes, Function> scriptFunction) { super(indexSettings); this.fieldName = fieldName; this.memorySizeInBytes = memorySizeInBytes; + this.ordinalMap = ordinalMap; + this.atomicReaders = new Atomic[segmentAfd.length]; + for (int i = 0; i < segmentAfd.length; i++) { + atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i); + } + this.scriptFunction = scriptFunction; } @Override @@ -88,4 +106,57 @@ public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponen // TODO: break down ram usage? return Collections.emptyList(); } + + @Override + public AtomicOrdinalsFieldData load(LeafReaderContext context) { + return atomicReaders[context.ord]; + } + + @Override + public MultiDocValues.OrdinalMap getOrdinalMap() { + return ordinalMap; + } + + private final class Atomic extends AbstractAtomicOrdinalsFieldData { + + private final AtomicOrdinalsFieldData afd; + private final MultiDocValues.OrdinalMap ordinalMap; + private final int segmentIndex; + + private Atomic(AtomicOrdinalsFieldData afd, MultiDocValues.OrdinalMap ordinalMap, int segmentIndex) { + super(scriptFunction); + this.afd = afd; + this.ordinalMap = ordinalMap; + this.segmentIndex = segmentIndex; + } + + @Override + public SortedSetDocValues getOrdinalsValues() { + final SortedSetDocValues values = afd.getOrdinalsValues(); + if (values.getValueCount() == ordinalMap.getValueCount()) { + // segment ordinals match global ordinals + return values; + } + final SortedSetDocValues[] bytesValues = new SortedSetDocValues[atomicReaders.length]; + for (int i = 0; i < bytesValues.length; i++) { + bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues(); + } + return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex); + } + + @Override + public long ramBytesUsed() { + return afd.ramBytesUsed(); + } + + @Override + public Collection getChildResources() { + return afd.getChildResources(); + } + + @Override + public void close() { + } + + } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java deleted file mode 100644 index 23ecc06fed6..00000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata.ordinals; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiDocValues.OrdinalMap; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.Accountable; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData; - -import java.util.Collection; -import java.util.function.Function; - -/** - * {@link org.elasticsearch.index.fielddata.IndexFieldData} impl based on global ordinals. - */ -final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFieldData { - - private final Atomic[] atomicReaders; - private final Function> scriptFunction; - - InternalGlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd, - OrdinalMap ordinalMap, long memorySizeInBytes, Function> scriptFunction) { - super(indexSettings, fieldName, memorySizeInBytes); - this.atomicReaders = new Atomic[segmentAfd.length]; - for (int i = 0; i < segmentAfd.length; i++) { - atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i); - } - this.scriptFunction = scriptFunction; - } - - @Override - public AtomicOrdinalsFieldData load(LeafReaderContext context) { - return atomicReaders[context.ord]; - } - - private final class Atomic extends AbstractAtomicOrdinalsFieldData { - - private final AtomicOrdinalsFieldData afd; - private final OrdinalMap ordinalMap; - private final int segmentIndex; - - private Atomic(AtomicOrdinalsFieldData afd, OrdinalMap ordinalMap, int segmentIndex) { - super(scriptFunction); - this.afd = afd; - this.ordinalMap = ordinalMap; - this.segmentIndex = segmentIndex; - } - - @Override - public SortedSetDocValues getOrdinalsValues() { - final SortedSetDocValues values = afd.getOrdinalsValues(); - if (values.getValueCount() == ordinalMap.getValueCount()) { - // segment ordinals match global ordinals - return values; - } - final SortedSetDocValues[] bytesValues = new SortedSetDocValues[atomicReaders.length]; - for (int i = 0; i < bytesValues.length; i++) { - bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues(); - } - return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex); - } - - @Override - public long ramBytesUsed() { - return afd.ramBytesUsed(); - } - - @Override - public Collection getChildResources() { - return afd.getChildResources(); - } - - @Override - public void close() { - } - - } - -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java deleted file mode 100644 index 2df5aa6bb63..00000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Set; - -import static java.util.Collections.emptySet; - - -abstract class AbstractAtomicParentChildFieldData implements AtomicParentChildFieldData { - - @Override - public final ScriptDocValues getScriptValues() { - return new ScriptDocValues.Strings(getBytesValues()); - } - - @Override - public final SortedBinaryDocValues getBytesValues() { - return new SortedBinaryDocValues() { - - private final SortedDocValues[] perTypeValues; - private final BytesRef[] terms = new BytesRef[2]; - private int count; - private int termsCursor; - - { - Set types = types(); - perTypeValues = new SortedDocValues[types.size()]; - int i = 0; - for (String type : types) { - perTypeValues[i++] = getOrdinalsValues(type); - } - } - - @Override - public boolean advanceExact(int docId) throws IOException { - count = 0; - termsCursor = 0; - - for (SortedDocValues values : perTypeValues) { - if (values.advanceExact(docId)) { - final int ord = values.ordValue(); - terms[count++] = values.lookupOrd(ord); - } - } - assert count <= 2 : "A single doc can potentially be both parent and child, so the maximum allowed values is 2"; - if (count > 1) { - int cmp = terms[0].compareTo(terms[1]); - if (cmp > 0) { - ArrayUtil.swap(terms, 0, 1); - } else if (cmp == 0) { - // If the id is the same between types the only omit one. For example: a doc has parent#1 in _uid field and has grand_parent#1 in _parent field. - count = 1; - } - } - return count != 0; - } - - @Override - public int docValueCount() { - return count; - } - - @Override - public BytesRef nextValue() throws IOException { - return terms[termsCursor++]; - } - }; - } - - public static AtomicParentChildFieldData empty() { - return new AbstractAtomicParentChildFieldData() { - - @Override - public long ramBytesUsed() { - return 0; - } - - @Override - public Collection getChildResources() { - return Collections.emptyList(); - } - - @Override - public void close() { - } - - @Override - public SortedDocValues getOrdinalsValues(String type) { - return DocValues.emptySorted(); - } - - @Override - public Set types() { - return emptySet(); - } - }; - } -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java index ce05e226b7e..1dbd082f93b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -51,6 +52,11 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD this.minSegmentSize = minSegmentSize; } + @Override + public MultiDocValues.OrdinalMap getOrdinalMap() { + return null; + } + @Override public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) { if (indexReader.leaves().size() <= 1) { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java deleted file mode 100644 index 74b180f2c1b..00000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java +++ /dev/null @@ -1,401 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiDocValues; -import org.apache.lucene.index.MultiDocValues.OrdinalMap; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.search.SortField; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LongValues; -import org.apache.lucene.util.packed.PackedInts; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.lease.Releasable; -import org.elasticsearch.common.lease.Releasables; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.AbstractSortedDocValues; -import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.IndexParentChildFieldData; -import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParentFieldMapper; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -/** - * ParentChildIndexFieldData is responsible for loading the id cache mapping - * needed for has_child and has_parent queries into memory. - */ -public class ParentChildIndexFieldData extends AbstractIndexFieldData implements IndexParentChildFieldData { - - private final Set parentTypes; - private final CircuitBreakerService breakerService; - - public ParentChildIndexFieldData(IndexSettings indexSettings, String fieldName, - IndexFieldDataCache cache, MapperService mapperService, - CircuitBreakerService breakerService) { - super(indexSettings, fieldName, cache); - this.breakerService = breakerService; - Set parentTypes = new HashSet<>(); - for (DocumentMapper mapper : mapperService.docMappers(false)) { - ParentFieldMapper parentFieldMapper = mapper.parentFieldMapper(); - if (parentFieldMapper.active()) { - parentTypes.add(parentFieldMapper.type()); - } - } - this.parentTypes = parentTypes; - } - - @Override - public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { - final XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); - return new SortField(getFieldName(), source, reverse); - } - - @Override - public AtomicParentChildFieldData load(LeafReaderContext context) { - final LeafReader reader = context.reader(); - return new AbstractAtomicParentChildFieldData() { - - public Set types() { - return parentTypes; - } - - @Override - public SortedDocValues getOrdinalsValues(String type) { - try { - return DocValues.getSorted(reader, ParentFieldMapper.joinField(type)); - } catch (IOException e) { - throw new IllegalStateException("cannot load join doc values field for type [" + type + "]", e); - } - } - - @Override - public long ramBytesUsed() { - // unknown - return 0; - } - - @Override - public Collection getChildResources() { - return Collections.emptyList(); - } - - @Override - public void close() throws ElasticsearchException { - } - }; - } - - @Override - public AbstractAtomicParentChildFieldData loadDirect(LeafReaderContext context) throws Exception { - throw new UnsupportedOperationException(); - } - - @Override - protected AtomicParentChildFieldData empty(int maxDoc) { - return AbstractAtomicParentChildFieldData.empty(); - } - - public static class Builder implements IndexFieldData.Builder { - - @Override - public IndexFieldData build(IndexSettings indexSettings, - MappedFieldType fieldType, - IndexFieldDataCache cache, CircuitBreakerService breakerService, - MapperService mapperService) { - return new ParentChildIndexFieldData(indexSettings, fieldType.name(), cache, - mapperService, breakerService); - } - } - - @Override - public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) { - if (indexReader.leaves().size() <= 1) { - // ordinals are already global - return this; - } - try { - return cache.load(indexReader, this); - } catch (Exception e) { - if (e instanceof ElasticsearchException) { - throw (ElasticsearchException) e; - } else { - throw new ElasticsearchException(e); - } - } - } - - private static OrdinalMap buildOrdinalMap(AtomicParentChildFieldData[] atomicFD, String parentType) throws IOException { - final SortedDocValues[] ordinals = new SortedDocValues[atomicFD.length]; - for (int i = 0; i < ordinals.length; ++i) { - ordinals[i] = atomicFD[i].getOrdinalsValues(parentType); - } - return OrdinalMap.build(null, ordinals, PackedInts.DEFAULT); - } - - private static class OrdinalMapAndAtomicFieldData { - final OrdinalMap ordMap; - final AtomicParentChildFieldData[] fieldData; - - OrdinalMapAndAtomicFieldData(OrdinalMap ordMap, AtomicParentChildFieldData[] fieldData) { - this.ordMap = ordMap; - this.fieldData = fieldData; - } - } - - @Override - public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception { - final long startTime = System.nanoTime(); - - long ramBytesUsed = 0; - final Map perType = new HashMap<>(); - for (String type : parentTypes) { - final AtomicParentChildFieldData[] fieldData = new AtomicParentChildFieldData[indexReader.leaves().size()]; - for (LeafReaderContext context : indexReader.leaves()) { - fieldData[context.ord] = load(context); - } - final OrdinalMap ordMap = buildOrdinalMap(fieldData, type); - ramBytesUsed += ordMap.ramBytesUsed(); - perType.put(type, new OrdinalMapAndAtomicFieldData(ordMap, fieldData)); - } - - final AtomicParentChildFieldData[] fielddata = new AtomicParentChildFieldData[indexReader.leaves().size()]; - for (int i = 0; i < fielddata.length; ++i) { - fielddata[i] = new GlobalAtomicFieldData(parentTypes, perType, i); - } - - breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(ramBytesUsed); - if (logger.isDebugEnabled()) { - logger.debug( - "global-ordinals [_parent] took [{}]", - new TimeValue(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) - ); - } - - return new GlobalFieldData(indexReader, fielddata, ramBytesUsed, perType); - } - - private static class GlobalAtomicFieldData extends AbstractAtomicParentChildFieldData { - - private final Set types; - private final Map atomicFD; - private final int segmentIndex; - - GlobalAtomicFieldData(Set types, Map atomicFD, int segmentIndex) { - this.types = types; - this.atomicFD = atomicFD; - this.segmentIndex = segmentIndex; - } - - @Override - public Set types() { - return types; - } - - @Override - public SortedDocValues getOrdinalsValues(String type) { - final OrdinalMapAndAtomicFieldData atomicFD = this.atomicFD.get(type); - if (atomicFD == null) { - return DocValues.emptySorted(); - } - - final OrdinalMap ordMap = atomicFD.ordMap; - final SortedDocValues[] allSegmentValues = new SortedDocValues[atomicFD.fieldData.length]; - for (int i = 0; i < allSegmentValues.length; ++i) { - allSegmentValues[i] = atomicFD.fieldData[i].getOrdinalsValues(type); - } - final SortedDocValues segmentValues = allSegmentValues[segmentIndex]; - if (segmentValues.getValueCount() == ordMap.getValueCount()) { - // ords are already global - return segmentValues; - } - final LongValues globalOrds = ordMap.getGlobalOrds(segmentIndex); - return new AbstractSortedDocValues() { - - @Override - public BytesRef lookupOrd(int ord) throws IOException { - final int segmentIndex = ordMap.getFirstSegmentNumber(ord); - final int segmentOrd = (int) ordMap.getFirstSegmentOrd(ord); - return allSegmentValues[segmentIndex].lookupOrd(segmentOrd); - } - - @Override - public int getValueCount() { - return (int) ordMap.getValueCount(); - } - - @Override - public int ordValue() throws IOException { - return (int) globalOrds.get(segmentValues.ordValue()); - } - - @Override - public boolean advanceExact(int target) throws IOException { - return segmentValues.advanceExact(target); - } - - @Override - public int docID() { - return segmentValues.docID(); - } - }; - } - - @Override - public long ramBytesUsed() { - // this class does not take memory on its own, the index-level field data does - // it through the use of ordinal maps - return 0; - } - - @Override - public Collection getChildResources() { - return Collections.emptyList(); - } - - @Override - public void close() { - List closeables = new ArrayList<>(); - for (OrdinalMapAndAtomicFieldData fds : atomicFD.values()) { - closeables.addAll(Arrays.asList(fds.fieldData)); - } - Releasables.close(closeables); - } - - } - - public class GlobalFieldData implements IndexParentChildFieldData, Accountable { - - private final Object coreCacheKey; - private final List leaves; - private final AtomicParentChildFieldData[] fielddata; - private final long ramBytesUsed; - private final Map ordinalMapPerType; - - GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed, Map ordinalMapPerType) { - this.coreCacheKey = reader.getReaderCacheHelper().getKey(); - this.leaves = reader.leaves(); - this.ramBytesUsed = ramBytesUsed; - this.fielddata = fielddata; - this.ordinalMapPerType = ordinalMapPerType; - } - - @Override - public String getFieldName() { - return ParentChildIndexFieldData.this.getFieldName(); - } - - @Override - public AtomicParentChildFieldData load(LeafReaderContext context) { - assert context.reader().getCoreCacheHelper().getKey() == leaves.get(context.ord) - .reader().getCoreCacheHelper().getKey(); - return fielddata[context.ord]; - } - - @Override - public AtomicParentChildFieldData loadDirect(LeafReaderContext context) throws Exception { - return load(context); - } - - @Override - public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { - throw new UnsupportedOperationException("No sorting on global ords"); - } - - @Override - public void clear() { - ParentChildIndexFieldData.this.clear(); - } - - @Override - public Index index() { - return ParentChildIndexFieldData.this.index(); - } - - @Override - public long ramBytesUsed() { - return ramBytesUsed; - } - - @Override - public Collection getChildResources() { - return Collections.emptyList(); - } - - @Override - public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) { - if (indexReader.getReaderCacheHelper().getKey() == coreCacheKey) { - return this; - } - throw new IllegalStateException(); - } - - @Override - public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception { - return loadGlobal(indexReader); - } - - } - - /** - * Returns the global ordinal map for the specified type - */ - // TODO: OrdinalMap isn't expose in the field data framework, because it is an implementation detail. - // However the JoinUtil works directly with OrdinalMap, so this is a hack to get access to OrdinalMap - // I don't think we should expose OrdinalMap in IndexFieldData, because only parent/child relies on it and for the - // rest of the code OrdinalMap is an implementation detail, but maybe we can expose it in IndexParentChildFieldData interface? - public static MultiDocValues.OrdinalMap getOrdinalMap(IndexParentChildFieldData indexParentChildFieldData, String type) { - if (indexParentChildFieldData instanceof ParentChildIndexFieldData.GlobalFieldData) { - return ((GlobalFieldData) indexParentChildFieldData).ordinalMapPerType.get(type).ordMap; - } else { - // one segment, local ordinals are global - return null; - } - } - -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java index ea076d476de..9e6e2e994c9 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedSetSelector; @@ -125,4 +126,9 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception { return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger, scriptFunction); } + + @Override + public MultiDocValues.OrdinalMap getOrdinalMap() { + return null; + } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java index 62877567c3d..3ef9b73708f 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java @@ -37,7 +37,7 @@ import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; +import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; @@ -196,7 +196,7 @@ public class ParentFieldMapper extends MetadataFieldMapper { @Override public IndexFieldData.Builder fielddataBuilder() { - return new ParentChildIndexFieldData.Builder(); + return new DocValuesIndexFieldData.Builder(); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index e6ed81c2d83..a8aaa259408 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -22,7 +22,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.IndexSearcher; @@ -32,18 +31,15 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.ScorerAware; import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; -import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; -import org.elasticsearch.index.fielddata.IndexParentChildFieldData; import org.elasticsearch.index.fielddata.MultiGeoPointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingBinaryDocValues; import org.elasticsearch.index.fielddata.SortingNumericDoubleValues; -import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.aggregations.support.ValuesSource.WithScript.BytesValues; @@ -150,40 +146,6 @@ public abstract class ValuesSource { } } - public static class ParentChild extends Bytes { - - protected final ParentChildIndexFieldData indexFieldData; - - public ParentChild(ParentChildIndexFieldData indexFieldData) { - this.indexFieldData = indexFieldData; - } - - public long globalMaxOrd(IndexSearcher indexSearcher, String type) { - DirectoryReader indexReader = (DirectoryReader) indexSearcher.getIndexReader(); - if (indexReader.leaves().isEmpty()) { - return 0; - } else { - LeafReaderContext atomicReaderContext = indexReader.leaves().get(0); - IndexParentChildFieldData globalFieldData = indexFieldData.loadGlobal(indexReader); - AtomicParentChildFieldData afd = globalFieldData.load(atomicReaderContext); - SortedDocValues values = afd.getOrdinalsValues(type); - return values.getValueCount(); - } - } - - public SortedDocValues globalOrdinalsValues(String type, LeafReaderContext context) { - final IndexParentChildFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader()); - final AtomicParentChildFieldData atomicFieldData = global.load(context); - return atomicFieldData.getOrdinalsValues(type); - } - - @Override - public SortedBinaryDocValues bytesValues(LeafReaderContext context) { - final AtomicParentChildFieldData atomicFieldData = indexFieldData.load(context); - return atomicFieldData.getBytesValues(); - } - } - public static class FieldData extends Bytes { protected final IndexFieldData indexFieldData; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index 39b97a9b025..e5fac62840f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -26,7 +26,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; -import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; @@ -317,9 +316,7 @@ public class ValuesSourceConfig { private ValuesSource bytesField() throws IOException { final IndexFieldData indexFieldData = fieldContext().indexFieldData(); ValuesSource dataSource; - if (indexFieldData instanceof ParentChildIndexFieldData) { - dataSource = new ValuesSource.Bytes.WithOrdinals.ParentChild((ParentChildIndexFieldData) indexFieldData); - } else if (indexFieldData instanceof IndexOrdinalsFieldData) { + if (indexFieldData instanceof IndexOrdinalsFieldData) { dataSource = new ValuesSource.Bytes.WithOrdinals.FieldData((IndexOrdinalsFieldData) indexFieldData); } else { dataSource = new ValuesSource.Bytes.FieldData(indexFieldData); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java index aebb0a802ed..05117550ac7 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java @@ -457,6 +457,7 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI refreshReader(); IndexOrdinalsFieldData ifd = getForField("string", "value", hasDocValues()); IndexOrdinalsFieldData globalOrdinals = ifd.loadGlobal(topLevelReader); + assertNotNull(globalOrdinals.getOrdinalMap()); assertThat(topLevelReader.leaves().size(), equalTo(3)); // First segment @@ -584,6 +585,7 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI refreshReader(); IndexOrdinalsFieldData ifd = getForField("string", "value", hasDocValues()); IndexOrdinalsFieldData globalOrdinals = ifd.loadGlobal(topLevelReader); + assertNotNull(globalOrdinals.getOrdinalMap()); assertThat(ifd.loadGlobal(topLevelReader), sameInstance(globalOrdinals)); // 3 b/c 1 segment level caches and 1 top level cache // in case of doc values, we don't cache atomic FD, so only the top-level cache is there diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java deleted file mode 100644 index f7bdc8efed1..00000000000 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.SortedDocValuesField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParentFieldMapper; -import org.elasticsearch.index.mapper.Uid; -import org.elasticsearch.index.mapper.UidFieldMapper; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.MultiValueMode; -import org.junit.Before; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicReference; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.nullValue; - -public class ParentChildFieldDataTests extends AbstractFieldDataTestCase { - private final String parentType = "parent"; - private final String childType = "child"; - private final String grandChildType = "grand-child"; - - @Before - public void setupData() throws Exception { - mapperService.merge( - childType, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(childType, "_parent", "type=" + parentType).string()), MapperService.MergeReason.MAPPING_UPDATE, false - ); - mapperService.merge( - grandChildType, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(grandChildType, "_parent", "type=" + childType).string()), MapperService.MergeReason.MAPPING_UPDATE, false - ); - - Document d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); - d.add(createJoinField(parentType, "1")); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "2"), Field.Store.NO)); - d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); - d.add(createJoinField(parentType, "1")); - d.add(createJoinField(childType, "2")); - writer.addDocument(d); - writer.commit(); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "3"), Field.Store.NO)); - d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); - d.add(createJoinField(parentType, "1")); - d.add(createJoinField(childType, "3")); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(parentType, "2"), Field.Store.NO)); - d.add(createJoinField(parentType, "2")); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "4"), Field.Store.NO)); - d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "2"), Field.Store.NO)); - d.add(createJoinField(parentType, "2")); - d.add(createJoinField(childType, "4")); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "5"), Field.Store.NO)); - d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); - d.add(createJoinField(parentType, "1")); - d.add(createJoinField(childType, "5")); - writer.addDocument(d); - writer.commit(); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(grandChildType, "6"), Field.Store.NO)); - d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(childType, "2"), Field.Store.NO)); - d.add(createJoinField(childType, "2")); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid("other-type", "1"), Field.Store.NO)); - writer.addDocument(d); - } - - private SortedDocValuesField createJoinField(String parentType, String id) { - return new SortedDocValuesField(ParentFieldMapper.joinField(parentType), new BytesRef(id)); - } - - public void testGetBytesValues() throws Exception { - writer.forceMerge(1); // force merge to 1 segment so we can iterate through documents - IndexFieldData indexFieldData = getForField(childType); - List readerContexts = refreshReader(); - for (LeafReaderContext readerContext : readerContexts) { - AtomicFieldData fieldData = indexFieldData.load(readerContext); - - SortedBinaryDocValues bytesValues = fieldData.getBytesValues(); - assertTrue(bytesValues.advanceExact(0)); - assertThat(bytesValues.docValueCount(), equalTo(1)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); - - assertTrue(bytesValues.advanceExact(1)); - assertThat(bytesValues.docValueCount(), equalTo(2)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - - assertTrue(bytesValues.advanceExact(2)); - assertThat(bytesValues.docValueCount(), equalTo(2)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("3")); - - assertTrue(bytesValues.advanceExact(3)); - assertThat(bytesValues.docValueCount(), equalTo(1)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - - assertTrue(bytesValues.advanceExact(4)); - assertThat(bytesValues.docValueCount(), equalTo(2)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("4")); - - assertTrue(bytesValues.advanceExact(5)); - assertThat(bytesValues.docValueCount(), equalTo(2)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("5")); - - assertTrue(bytesValues.advanceExact(6)); - assertThat(bytesValues.docValueCount(), equalTo(1)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - - assertFalse(bytesValues.advanceExact(7)); - } - } - - public void testSorting() throws Exception { - IndexFieldData indexFieldData = getForField(parentType); - IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer)); - SortField sortField = indexFieldData.sortField("_last", MultiValueMode.MIN, null, false); - TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField)); - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[1].doc, equalTo(1)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[3].doc, equalTo(5)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[4].doc, equalTo(3)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[5].doc, equalTo(4)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[6].doc, equalTo(6)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[7].doc, equalTo(7)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]), equalTo(null)); - - sortField = indexFieldData.sortField("_last", MultiValueMode.MIN, null, true); - topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField)); - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(3)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[1].doc, equalTo(4)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[2].doc, equalTo(6)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[3].doc, equalTo(0)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[4].doc, equalTo(1)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[5].doc, equalTo(2)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[6].doc, equalTo(5)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[7].doc, equalTo(7)); - assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], nullValue()); - } - - public void testThreads() throws Exception { - final ParentChildIndexFieldData indexFieldData = getForField(childType); - final DirectoryReader reader = ElasticsearchDirectoryReader.wrap( - DirectoryReader.open(writer), new ShardId(new Index("test", ""), 0)); - final IndexParentChildFieldData global = indexFieldData.loadGlobal(reader); - final AtomicReference error = new AtomicReference<>(); - final int numThreads = scaledRandomIntBetween(3, 8); - final Thread[] threads = new Thread[numThreads]; - final CountDownLatch latch = new CountDownLatch(1); - - final Map expected = new HashMap<>(); - for (LeafReaderContext context : reader.leaves()) { - AtomicParentChildFieldData leafData = global.load(context); - SortedDocValues parentIds = leafData.getOrdinalsValues(parentType); - final BytesRef[] ids = new BytesRef[parentIds.getValueCount()]; - for (int j = 0; j < parentIds.getValueCount(); ++j) { - final BytesRef id = parentIds.lookupOrd(j); - if (id != null) { - ids[j] = BytesRef.deepCopyOf(id); - } - } - expected.put(context.reader().getCoreCacheHelper().getKey(), ids); - } - - for (int i = 0; i < numThreads; ++i) { - threads[i] = new Thread() { - @Override - public void run() { - try { - latch.await(); - for (int i = 0; i < 100000; ++i) { - for (LeafReaderContext context : reader.leaves()) { - AtomicParentChildFieldData leafData = global.load(context); - SortedDocValues parentIds = leafData.getOrdinalsValues(parentType); - final BytesRef[] expectedIds = expected.get(context.reader().getCoreCacheHelper().getKey()); - for (int j = 0; j < parentIds.getValueCount(); ++j) { - final BytesRef id = parentIds.lookupOrd(j); - assertEquals(expectedIds[j], id); - } - } - } - } catch (Exception e) { - error.compareAndSet(null, e); - } - } - }; - threads[i].start(); - } - latch.countDown(); - for (Thread thread : threads) { - thread.join(); - } - if (error.get() != null) { - throw error.get(); - } - } - - @Override - protected String getFieldDataType() { - return "_parent"; - } -} diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java index d04b1f0a660..35dc4eacbf3 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java @@ -25,15 +25,16 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; +import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.ParentFieldMapper; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.support.FieldContext; import org.elasticsearch.search.aggregations.support.ValueType; -import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.ParentChild; +import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.WithOrdinals; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; @@ -43,10 +44,11 @@ import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; import java.util.Objects; -public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder { +public class ChildrenAggregationBuilder + extends ValuesSourceAggregationBuilder { + public static final String NAME = "children"; - private String parentType; private final String childType; private Query parentFilter; private Query childFilter; @@ -79,15 +81,17 @@ public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder

innerBuild(SearchContext context, - ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { - return new ChildrenAggregatorFactory(name, config, parentType, childFilter, parentFilter, context, parent, + protected ValuesSourceAggregatorFactory innerBuild(SearchContext context, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder) throws IOException { + return new ChildrenAggregatorFactory(name, config, childFilter, parentFilter, context, parent, subFactoriesBuilder, metaData); } @Override - protected ValuesSourceConfig resolveConfig(SearchContext context) { - ValuesSourceConfig config = new ValuesSourceConfig<>(ValuesSourceType.BYTES); + protected ValuesSourceConfig resolveConfig(SearchContext context) { + ValuesSourceConfig config = new ValuesSourceConfig<>(ValuesSourceType.BYTES); DocumentMapper childDocMapper = context.mapperService().documentMapper(childType); if (childDocMapper != null) { @@ -95,15 +99,15 @@ public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder

{ + extends ValuesSourceAggregatorFactory { - private final String parentType; private final Query parentFilter; private final Query childFilter; - public ChildrenAggregatorFactory(String name, ValuesSourceConfig config, String parentType, Query childFilter, - Query parentFilter, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, - Map metaData) throws IOException { + public ChildrenAggregatorFactory(String name, ValuesSourceConfig config, + Query childFilter, Query parentFilter, SearchContext context, AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { super(name, config, context, parent, subFactoriesBuilder, metaData); - this.parentType = parentType; this.childFilter = childFilter; this.parentFilter = parentFilter; } @@ -66,12 +63,11 @@ public class ChildrenAggregatorFactory } @Override - protected Aggregator doCreateInternal(ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource, Aggregator parent, + protected Aggregator doCreateInternal(WithOrdinals valuesSource, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - long maxOrd = valuesSource.globalMaxOrd(context.searcher(), parentType); - return new ParentToChildrenAggregator(name, factories, context, parent, parentType, childFilter, parentFilter, valuesSource, maxOrd, - pipelineAggregators, metaData); + long maxOrd = valuesSource.globalMaxOrd(context.searcher()); + return new ParentToChildrenAggregator(name, factories, context, parent, childFilter, + parentFilter, valuesSource, maxOrd, pipelineAggregators, metaData); } - } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java index c1ffb097abc..93ba1b98da1 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java @@ -20,7 +20,7 @@ package org.elasticsearch.join.aggregations; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Query; @@ -52,10 +52,9 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { static final ParseField TYPE_FIELD = new ParseField("type"); - private final String parentType; private final Weight childFilter; private final Weight parentFilter; - private final ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource; + private final ValuesSource.Bytes.WithOrdinals valuesSource; // Maybe use PagedGrowableWriter? This will be less wasteful than LongArray, // but then we don't have the reuse feature of BigArrays. @@ -72,12 +71,11 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { private boolean multipleBucketsPerParentOrd = false; public ParentToChildrenAggregator(String name, AggregatorFactories factories, - SearchContext context, Aggregator parent, String parentType, Query childFilter, - Query parentFilter, ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource, + SearchContext context, Aggregator parent, Query childFilter, + Query parentFilter, ValuesSource.Bytes.WithOrdinals valuesSource, long maxOrd, List pipelineAggregators, Map metaData) throws IOException { super(name, factories, context, parent, pipelineAggregators, metaData); - this.parentType = parentType; // these two filters are cached in the parser this.childFilter = context.searcher().createNormalizedWeight(childFilter, false); this.parentFilter = context.searcher().createNormalizedWeight(parentFilter, false); @@ -105,9 +103,7 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { if (valuesSource == null) { return LeafBucketCollector.NO_OP_COLLECTOR; } - - final SortedDocValues globalOrdinals = valuesSource.globalOrdinalsValues(parentType, ctx); - assert globalOrdinals != null; + final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx); Scorer parentScorer = parentFilter.scorer(ctx); final Bits parentDocs = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), parentScorer); return new LeafBucketCollector() { @@ -115,7 +111,8 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { @Override public void collect(int docId, long bucket) throws IOException { if (parentDocs.get(docId) && globalOrdinals.advanceExact(docId)) { - long globalOrdinal = globalOrdinals.ordValue(); + long globalOrdinal = globalOrdinals.nextOrd(); + assert globalOrdinals.nextOrd() == SortedSetDocValues.NO_MORE_ORDS; if (globalOrdinal != -1) { if (parentOrdToBuckets.get(globalOrdinal) == -1) { parentOrdToBuckets.set(globalOrdinal, bucket); @@ -147,9 +144,8 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { DocIdSetIterator childDocsIter = childDocsScorer.iterator(); final LeafBucketCollector sub = collectableSubAggregators.getLeafCollector(ctx); - final SortedDocValues globalOrdinals = valuesSource.globalOrdinalsValues(parentType, - ctx); + final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx); // Set the scorer, since we now replay only the child docIds sub.setScorer(new ConstantScoreScorer(null, 1f, childDocsIter)); @@ -161,7 +157,8 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { continue; } if (globalOrdinals.advanceExact(docId)) { - long globalOrdinal = globalOrdinals.ordValue(); + long globalOrdinal = globalOrdinals.nextOrd(); + assert globalOrdinals.nextOrd() == SortedSetDocValues.NO_MORE_ORDS; long bucketOrd = parentOrdToBuckets.get(globalOrdinal); if (bucketOrd != -1) { collectBucket(sub, docId, bucketOrd); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java index 494c5e498e1..95d000e3ccc 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java @@ -34,9 +34,10 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexParentChildFieldData; -import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; +import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.ParentFieldMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.InnerHitBuilder; @@ -48,7 +49,6 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import java.io.IOException; -import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -324,9 +324,10 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder childTypes = new HashSet<>(); - ParentChildIndexFieldData parentChildIndexFieldData = null; for (DocumentMapper documentMapper : context.getMapperService().docMappers(false)) { ParentFieldMapper parentFieldMapper = documentMapper.parentFieldMapper(); if (parentFieldMapper.active() && type.equals(parentFieldMapper.type())) { childTypes.add(documentMapper.type()); - parentChildIndexFieldData = context.getForField(parentFieldMapper.fieldType()); } } - if (childTypes.isEmpty()) { throw new QueryShardException(context, "[" + NAME + "] no child types found for type [" + type + "]"); } @@ -204,14 +202,17 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder verify) diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java index ed910ac89e6..87efdf0a5ca 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ChildQuerySearchIT.java @@ -105,7 +105,7 @@ import static org.hamcrest.Matchers.notNullValue; @ClusterScope(scope = Scope.SUITE) public class ChildQuerySearchIT extends ESIntegTestCase { - + @Override protected boolean ignoreExternalCluster() { return true; @@ -2008,7 +2008,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase { .setParent("parent-id").setSource("searchText", "quick brown fox").get(); refresh(); - String[] highlightTypes = new String[] {"plain", "fvh", "postings"}; + String[] highlightTypes = new String[] {"plain", "fvh", "unified"}; for (String highlightType : highlightTypes) { logger.info("Testing with highlight type [{}]", highlightType); SearchResponse searchResponse = client().prepareSearch("test")