diff --git a/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index 371cc42d003..b8633f47c4a 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -40,6 +40,13 @@ public enum FieldData { assert Lucene.VERSION == Version.LUCENE_4_9 : "Remove emptySortedNumeric in 4.10 and use the method with the same name from Lucene's DocValues class. See LUCENE-5834."; } + /** + * Return a {@link SortedBinaryDocValues} that doesn't contain any value. + */ + public static SortedBinaryDocValues emptySortedBinary(int maxDoc) { + return singleton(DocValues.emptyBinary(), new Bits.MatchNoBits(maxDoc)); + } + /** * Return a {@link SortedNumericDocValues} that doesn't contain any value. */ diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java index f6473647479..62faf5abce1 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java +++ b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java @@ -21,20 +21,23 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.FieldComparatorSource; -import org.apache.lucene.search.SortField; +import org.apache.lucene.search.*; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexComponent; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService; import org.elasticsearch.search.MultiValueMode; +import java.io.IOException; + /** * Thread-safe utility class that allows to get per-segment values via the * {@link #load(AtomicReaderContext)} method. @@ -93,7 +96,7 @@ public interface IndexFieldData extends IndexCompone /** * Comparator used for sorting. */ - XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode); + XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested); /** * Clears any resources associated with this field data. @@ -116,6 +119,52 @@ public interface IndexFieldData extends IndexCompone UnicodeUtil.UTF16toUTF8(chars, 0, chars.length, MAX_TERM); } + /** + * Simple wrapper class around a filter that matches parent documents + * and a filter that matches child documents. For every root document R, + * R will be in the parent filter and its children documents will be the + * documents that are contained in the inner set between the previous + * parent + 1, or 0 if there is no previous parent, and R (excluded). + */ + public static class Nested { + private final Filter rootFilter, innerFilter; + + public Nested(Filter rootFilter, Filter innerFilter) { + this.rootFilter = rootFilter; + this.innerFilter = innerFilter; + } + + // TODO: nested docs should not be random filters but specialized + // ones that guarantee that you always get a FixedBitSet + @Deprecated + private static FixedBitSet toFixedBitSet(DocIdSet set, int maxDoc) throws IOException { + if (set == null || set instanceof FixedBitSet) { + return (FixedBitSet) set; + } else { + final FixedBitSet fixedBitSet = new FixedBitSet(maxDoc); + final DocIdSetIterator it = set.iterator(); + if (it != null) { + fixedBitSet.or(it); + } + return fixedBitSet; + } + } + + /** + * Get a {@link FixedBitSet} that matches the root documents. + */ + public FixedBitSet rootDocs(AtomicReaderContext ctx) throws IOException { + return toFixedBitSet(rootFilter.getDocIdSet(ctx, null), ctx.reader().maxDoc()); + } + + /** + * Get a {@link FixedBitSet} that matches the inner documents. + */ + public FixedBitSet innerDocs(AtomicReaderContext ctx) throws IOException { + return toFixedBitSet(innerFilter.getDocIdSet(ctx, null), ctx.reader().maxDoc()); + } + } + /** Whether missing values should be sorted first. */ protected final boolean sortMissingFirst(Object missingValue) { return "_first".equals(missingValue); diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 02a3287c410..a4d2c273f50 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -19,37 +19,41 @@ package org.elasticsearch.index.fielddata.fieldcomparator; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.Scorer; import org.apache.lucene.search.SortField; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.Version; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.search.MultiValueMode; import java.io.IOException; /** + * Comparator source for string/binary values. */ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparatorSource { - /** UTF-8 term containing a single code point: {@link Character#MAX_CODE_POINT} which will compare greater than all other index terms - * since {@link Character#MAX_CODE_POINT} is a noncharacter and thus shouldn't appear in an index term. */ - public static final BytesRef MAX_TERM; - static { - MAX_TERM = new BytesRef(); - final char[] chars = Character.toChars(Character.MAX_CODE_POINT); - UnicodeUtil.UTF16toUTF8(chars, 0, chars.length, MAX_TERM); - } - private final IndexFieldData indexFieldData; private final MultiValueMode sortMode; private final Object missingValue; + private final Nested nested; - public BytesRefFieldComparatorSource(IndexFieldData indexFieldData, Object missingValue, MultiValueMode sortMode) { + public BytesRefFieldComparatorSource(IndexFieldData indexFieldData, Object missingValue, MultiValueMode sortMode, Nested nested) { this.indexFieldData = indexFieldData; this.sortMode = sortMode; this.missingValue = missingValue; + this.nested = nested; } @Override @@ -57,15 +61,229 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat return SortField.Type.STRING; } + protected SortedBinaryDocValues getValues(AtomicReaderContext context) { + return indexFieldData.load(context).getBytesValues(); + } + + protected void setScorer(Scorer scorer) {} + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - assert fieldname.equals(indexFieldData.getFieldNames().indexName()); - final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed); + assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName()); + final boolean sortMissingLast = sortMissingLast(missingValue) ^ reversed; + final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed); if (indexFieldData instanceof IndexOrdinalsFieldData) { - return new BytesRefOrdValComparator((IndexOrdinalsFieldData) indexFieldData, numHits, sortMode, missingBytes); + // The ordinal-based comparator only supports sorting missing values first or last so when + // a missing value is provided we fall back to the (slow) value-based comparator + // TODO: handle arbitrary missing values via a selector + if (sortMissingFirst(missingValue) || sortMissingLast(missingValue)) { + return new FieldComparator.TermOrdValComparator(numHits, null, sortMissingLast) { + + @Override + protected SortedDocValues getSortedDocValues(AtomicReaderContext context, String field) throws IOException { + final RandomAccessOrds values = ((IndexOrdinalsFieldData) indexFieldData).load(context).getOrdinalsValues(); + final SortedDocValues selectedValues; + if (nested == null) { + selectedValues = sortMode.select(values); + } else { + final FixedBitSet rootDocs = nested.rootDocs(context); + final FixedBitSet innerDocs = nested.innerDocs(context); + selectedValues = sortMode.select(values, rootDocs, innerDocs); + } + return selectedValues; + } + + public BytesRef value(int slot) { + // TODO: When serializing the response to the coordinating node, we lose the information about + // whether the comparator sorts missing docs first or last. We should fix it and let + // TopDocs.merge deal with it (it knows how to) + BytesRef value = super.value(slot); + if (value == null) { + value = missingBytes; + } + return value; + } + + }; + } } - return new BytesRefValComparator(indexFieldData, numHits, sortMode, missingBytes); + + final BytesRef nullPlaceHolder = new BytesRef(); + final BytesRef nonNullMissingBytes = missingBytes == null ? nullPlaceHolder : missingBytes; + return new TermValComparator(numHits, null, sortMissingLast) { + + @Override + protected BinaryDocValues getBinaryDocValues(AtomicReaderContext context, String field) throws IOException { + final SortedBinaryDocValues values = getValues(context); + final BinaryDocValues selectedValues; + if (nested == null) { + selectedValues = sortMode.select(values, nonNullMissingBytes); + } else { + final FixedBitSet rootDocs = nested.rootDocs(context); + final FixedBitSet innerDocs = nested.innerDocs(context); + selectedValues = sortMode.select(values, nonNullMissingBytes, rootDocs, innerDocs, context.reader().maxDoc()); + } + return selectedValues; + } + + @Override + protected Bits getDocsWithField(AtomicReaderContext context, String field) throws IOException { + return new Bits.MatchAllBits(context.reader().maxDoc()); + } + + @Override + protected boolean isNull(int doc, BytesRef term) { + return term == nullPlaceHolder; + } + + @Override + public void setScorer(Scorer scorer) { + BytesRefFieldComparatorSource.this.setScorer(scorer); + } + + @Override + public BytesRef value(int slot) { + BytesRef value = super.value(slot); + if (value == null) { + value = missingBytes; + } + return value; + } + + }; + } + + static { + assert Lucene.VERSION == Version.LUCENE_4_9 : "The comparator below is a raw copy of Lucene's, remove it when upgrading to 4.10"; + } + + /** Sorts by field's natural Term sort order. All + * comparisons are done using BytesRef.compareTo, which is + * slow for medium to large result sets but possibly + * very fast for very small results sets. */ + public static class TermValComparator extends FieldComparator { + + private final BytesRef[] values; + private final BytesRef[] tempBRs; + private BinaryDocValues docTerms; + private Bits docsWithField; + private final String field; + private BytesRef bottom; + private BytesRef topValue; + private final int missingSortCmp; + + /** Sole constructor. */ + public TermValComparator(int numHits, String field, boolean sortMissingLast) { + values = new BytesRef[numHits]; + tempBRs = new BytesRef[numHits]; + this.field = field; + missingSortCmp = sortMissingLast ? 1 : -1; + } + + @Override + public int compare(int slot1, int slot2) { + final BytesRef val1 = values[slot1]; + final BytesRef val2 = values[slot2]; + return compareValues(val1, val2); + } + + @Override + public int compareBottom(int doc) { + final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc)); + return compareValues(bottom, comparableBytes); + } + + @Override + public void copy(int slot, int doc) { + final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc)); + if (comparableBytes == null) { + values[slot] = null; + } else { + if (tempBRs[slot] == null) { + tempBRs[slot] = new BytesRef(); + } + values[slot] = tempBRs[slot]; + values[slot].copyBytes(comparableBytes); + } + } + + /** Retrieves the BinaryDocValues for the field in this segment */ + protected BinaryDocValues getBinaryDocValues(AtomicReaderContext context, String field) throws IOException { + return FieldCache.DEFAULT.getTerms(context.reader(), field, true); + } + + /** Retrieves the set of documents that have a value in this segment */ + protected Bits getDocsWithField(AtomicReaderContext context, String field) throws IOException { + return FieldCache.DEFAULT.getDocsWithField(context.reader(), field); + } + + /** Check whether the given value represents null. This can be + * useful if the {@link BinaryDocValues} returned by {@link #getBinaryDocValues} + * use a special value as a sentinel. The default implementation checks + * {@link #getDocsWithField}. + *

NOTE: The null value can only be an EMPTY {@link BytesRef}. */ + protected boolean isNull(int doc, BytesRef term) { + return docsWithField != null && docsWithField.get(doc) == false; + } + + @Override + public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { + docTerms = getBinaryDocValues(context, field); + docsWithField = getDocsWithField(context, field); + if (docsWithField instanceof Bits.MatchAllBits) { + docsWithField = null; + } + return this; + } + + @Override + public void setBottom(final int bottom) { + this.bottom = values[bottom]; + } + + @Override + public void setTopValue(BytesRef value) { + // null is fine: it means the last doc of the prior + // search was missing this value + topValue = value; + } + + @Override + public BytesRef value(int slot) { + return values[slot]; + } + + @Override + public int compareValues(BytesRef val1, BytesRef val2) { + // missing always sorts first: + if (val1 == null) { + if (val2 == null) { + return 0; + } + return missingSortCmp; + } else if (val2 == null) { + return -missingSortCmp; + } + return val1.compareTo(val2); + } + + @Override + public int compareTop(int doc) { + final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc)); + return compareValues(topValue, comparableBytes); + } + + /** + * Given a document and a term, return the term itself if it exists or + * null otherwise. + */ + private BytesRef getComparableBytes(int doc, BytesRef term) { + if (term.length == 0 && isNull(doc, term)) { + return null; + } + return term; + } } } diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java deleted file mode 100644 index 12eaf5ecccb..00000000000 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java +++ /dev/null @@ -1,339 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -/** - * Sorts by field's natural Term sort order, using - * ordinals. This is functionally equivalent to {@link - * org.apache.lucene.search.FieldComparator.TermValComparator}, but it first resolves the string - * to their relative ordinal positions (using the index - * returned by {@link org.apache.lucene.search.FieldCache#getTermsIndex}), and - * does most comparisons using the ordinals. For medium - * to large results, this comparator will be much faster - * than {@link org.apache.lucene.search.FieldComparator.TermValComparator}. For very small - * result sets it may be slower. - * - * Internally this comparator multiplies ordinals by 4 so that virtual ordinals can be inserted in-between the original field data ordinals. - * Thanks to this, an ordinal for the missing value and the bottom value can be computed and all ordinals are directly comparable. For example, - * if the field data ordinals are (a,1), (b,2) and (c,3), they will be internally stored as (a,4), (b,8), (c,12). Then the ordinal for the - * missing value will be computed by binary searching. For example, if the missing value is 'ab', it will be assigned 6 as an ordinal (between - * 'a' and 'b'. And if the bottom value is 'ac', it will be assigned 7 as an ordinal (between 'ab' and 'b'). - */ -public final class BytesRefOrdValComparator extends NestedWrappableComparator { - - final IndexOrdinalsFieldData indexFieldData; - final BytesRef missingValue; - - /* Ords for each slot, times 4. - @lucene.internal */ - final long[] ords; - - final MultiValueMode sortMode; - - /* Values for each slot. - @lucene.internal */ - final BytesRef[] values; - - /* Which reader last copied a value into the slot. When - we compare two slots, we just compare-by-ord if the - readerGen is the same; else we must compare the - values (slower). - @lucene.internal */ - final int[] readerGen; - - /* Gen of current reader we are on. - @lucene.internal */ - int currentReaderGen = -1; - - /* Current reader's doc ord/values. - @lucene.internal */ - SortedDocValues termsIndex; - long missingOrd; - - /* Bottom slot, or -1 if queue isn't full yet - @lucene.internal */ - int bottomSlot = -1; - - /* Bottom ord (same as ords[bottomSlot] once bottomSlot - is set). Cached for faster compares. - @lucene.internal */ - long bottomOrd; - - BytesRef top; - long topOrd; - - public BytesRefOrdValComparator(IndexOrdinalsFieldData indexFieldData, int numHits, MultiValueMode sortMode, BytesRef missingValue) { - this.indexFieldData = indexFieldData; - this.sortMode = sortMode; - this.missingValue = missingValue; - ords = new long[numHits]; - values = new BytesRef[numHits]; - readerGen = new int[numHits]; - } - - @Override - public int compare(int slot1, int slot2) { - if (readerGen[slot1] == readerGen[slot2]) { - final int res = Long.compare(ords[slot1], ords[slot2]); - assert Integer.signum(res) == Integer.signum(compareValues(values[slot1], values[slot2])) : values[slot1] + " " + values[slot2] + " " + ords[slot1] + " " + ords[slot2]; - return res; - } - - final BytesRef val1 = values[slot1]; - final BytesRef val2 = values[slot2]; - return compareValues(val1, val2); - } - - @Override - public int compareBottom(int doc) { - throw new UnsupportedOperationException(); - } - - @Override - public int compareTop(int doc) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public int compareBottomMissing() { - throw new UnsupportedOperationException(); - } - - @Override - public void copy(int slot, int doc) { - throw new UnsupportedOperationException(); - } - - @Override - public void missing(int slot) { - throw new UnsupportedOperationException(); - } - - @Override - public int compareTopMissing() { - throw new UnsupportedOperationException(); - } - - class PerSegmentComparator extends NestedWrappableComparator { - final SortedDocValues termsIndex; - - public PerSegmentComparator(SortedDocValues termsIndex) { - this.termsIndex = termsIndex; - } - - @Override - public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - return BytesRefOrdValComparator.this.setNextReader(context); - } - - @Override - public int compare(int slot1, int slot2) { - return BytesRefOrdValComparator.this.compare(slot1, slot2); - } - - @Override - public void setBottom(final int bottom) { - BytesRefOrdValComparator.this.setBottom(bottom); - } - - @Override - public void setTopValue(BytesRef value) { - BytesRefOrdValComparator.this.setTopValue(value); - } - - @Override - public BytesRef value(int slot) { - return BytesRefOrdValComparator.this.value(slot); - } - - @Override - public int compareValues(BytesRef val1, BytesRef val2) { - if (val1 == null) { - if (val2 == null) { - return 0; - } - return -1; - } else if (val2 == null) { - return 1; - } - return val1.compareTo(val2); - } - - @Override - public int compareBottom(int doc) { - assert bottomSlot != -1; - final long docOrd = termsIndex.getOrd(doc); - final long comparableOrd = docOrd < 0 ? missingOrd : docOrd << 2; - return Long.compare(bottomOrd, comparableOrd); - } - - @Override - public int compareTop(int doc) throws IOException { - final long ord = termsIndex.getOrd(doc); - if (ord < 0) { - return compareTopMissing(); - } else { - final long comparableOrd = ord << 2; - return Long.compare(topOrd, comparableOrd); - } - } - - @Override - public int compareBottomMissing() { - assert bottomSlot != -1; - return Long.compare(bottomOrd, missingOrd); - } - - @Override - public int compareTopMissing() { - int cmp = Long.compare(topOrd, missingOrd); - if (cmp == 0) { - return compareValues(top, missingValue); - } else { - return cmp; - } - } - - @Override - public void copy(int slot, int doc) { - final int ord = termsIndex.getOrd(doc); - if (ord < 0) { - ords[slot] = missingOrd; - values[slot] = missingValue; - } else { - assert ord >= 0; - ords[slot] = ((long) ord) << 2; - if (values[slot] == null || values[slot] == missingValue) { - values[slot] = new BytesRef(); - } - values[slot].copyBytes(termsIndex.lookupOrd(ord)); - } - readerGen[slot] = currentReaderGen; - } - - @Override - public void missing(int slot) { - ords[slot] = missingOrd; - values[slot] = missingValue; - readerGen[slot] = currentReaderGen; - } - } - - // for assertions - private boolean consistentInsertedOrd(SortedDocValues termsIndex, long ord, BytesRef value) { - final int previousOrd = (int) (ord >> 2); - final int nextOrd = previousOrd + 1; - final BytesRef previous = previousOrd < 0 ? null : termsIndex.lookupOrd(previousOrd); - if ((ord & 3) == 0) { // there was an existing ord with the inserted value - assert compareValues(previous, value) == 0; - } else { - assert compareValues(previous, value) < 0; - } - if (nextOrd < termsIndex.getValueCount()) { - final BytesRef next = termsIndex.lookupOrd(nextOrd); - assert compareValues(value, next) < 0; - } - return true; - } - - // find where to insert an ord in the current terms index - private long ordInCurrentReader(SortedDocValues termsIndex, BytesRef value) { - final long ord; - if (value == null) { - ord = -1 << 2; - } else { - final long docOrd = termsIndex.lookupTerm(value); - if (docOrd >= 0) { - // value exists in the current segment - ord = docOrd << 2; - } else { - // value doesn't exist, use the ord between the previous and the next term - ord = ((-2 - docOrd) << 2) + 2; - } - } - - assert (ord & 1) == 0; - return ord; - } - - @Override - public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - termsIndex = sortMode.select(indexFieldData.load(context).getOrdinalsValues(), -1); - missingOrd = ordInCurrentReader(termsIndex, missingValue); - assert consistentInsertedOrd(termsIndex, missingOrd, missingValue); - FieldComparator perSegComp = new PerSegmentComparator(termsIndex); - currentReaderGen++; - if (bottomSlot != -1) { - perSegComp.setBottom(bottomSlot); - } - if (top != null) { - perSegComp.setTopValue(top); - topOrd = ordInCurrentReader(termsIndex, top); - } else { - topOrd = missingOrd; - } - return perSegComp; - } - - @Override - public void setBottom(final int bottom) { - bottomSlot = bottom; - final BytesRef bottomValue = values[bottomSlot]; - - if (currentReaderGen == readerGen[bottomSlot]) { - bottomOrd = ords[bottomSlot]; - } else { - // insert an ord - bottomOrd = ordInCurrentReader(termsIndex, bottomValue); - if (bottomOrd == missingOrd && bottomValue != null) { - // bottomValue and missingValue and in-between the same field data values -> tie-break - // this is why we multiply ords by 4 - assert missingValue != null; - final int cmp = bottomValue.compareTo(missingValue); - if (cmp < 0) { - --bottomOrd; - } else if (cmp > 0) { - ++bottomOrd; - } - } - assert consistentInsertedOrd(termsIndex, bottomOrd, bottomValue); - } - } - - @Override - public void setTopValue(BytesRef value) { - this.top = value; - } - - @Override - public BytesRef value(int slot) { - return values[slot]; - } - -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java deleted file mode 100644 index b70e4dffdb4..00000000000 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -/** - * Sorts by field's natural Term sort order. All - * comparisons are done using BytesRef.compareTo, which is - * slow for medium to large result sets but possibly - * very fast for very small results sets. - */ -public final class BytesRefValComparator extends NestedWrappableComparator { - - private final IndexFieldData indexFieldData; - private final MultiValueMode sortMode; - private final BytesRef missingValue; - - private final BytesRef[] values; - private BytesRef bottom; - private BytesRef top; - private BinaryDocValues docTerms; - - BytesRefValComparator(IndexFieldData indexFieldData, int numHits, MultiValueMode sortMode, BytesRef missingValue) { - this.sortMode = sortMode; - values = new BytesRef[numHits]; - this.indexFieldData = indexFieldData; - this.missingValue = missingValue; - } - - @Override - public int compare(int slot1, int slot2) { - final BytesRef val1 = values[slot1]; - final BytesRef val2 = values[slot2]; - return compareValues(val1, val2); - } - - @Override - public int compareBottom(int doc) throws IOException { - BytesRef val2 = docTerms.get(doc); - return compareValues(bottom, val2); - } - - @Override - public int compareTop(int doc) throws IOException { - return top.compareTo(docTerms.get(doc)); - } - - @Override - public void copy(int slot, int doc) throws IOException { - BytesRef relevantValue = docTerms.get(doc); - if (relevantValue == missingValue) { - values[slot] = missingValue; - } else { - if (values[slot] == null || values[slot] == missingValue) { - values[slot] = new BytesRef(); - } - values[slot].copyBytes(relevantValue); - } - } - - @Override - public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - final SortedBinaryDocValues docTerms = indexFieldData.load(context).getBytesValues(); - this.docTerms = sortMode.select(docTerms, missingValue); - return this; - } - - @Override - public void setBottom(final int bottom) { - this.bottom = values[bottom]; - } - - @Override - public void setTopValue(BytesRef top) { - this.top = top; - } - - @Override - public BytesRef value(int slot) { - return values[slot]; - } - - @Override - public int compareValues(BytesRef val1, BytesRef val2) { - if (val1 == null) { - if (val2 == null) { - return 0; - } - return -1; - } else if (val2 == null) { - return 1; - } - return val1.compareTo(val2); - } - - @Override - public void missing(int slot) { - values[slot] = missingValue; - } - - @Override - public int compareBottomMissing() { - return compareValues(bottom, missingValue); - } - - @Override - public int compareTopMissing() { - return compareValues(top, missingValue); - } -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleScriptDataComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleScriptDataComparator.java deleted file mode 100644 index 410840e64ba..00000000000 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleScriptDataComparator.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SortField; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.script.SearchScript; - -import java.io.IOException; - -/** - * - */ -// LUCENE MONITOR: Monitor against FieldComparator.Double -public class DoubleScriptDataComparator extends NumberComparatorBase { - - public static IndexFieldData.XFieldComparatorSource comparatorSource(SearchScript script) { - return new InnerSource(script); - } - - private static class InnerSource extends IndexFieldData.XFieldComparatorSource { - - private final SearchScript script; - - private InnerSource(SearchScript script) { - this.script = script; - } - - @Override - public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - return new DoubleScriptDataComparator(numHits, script); - } - - @Override - public SortField.Type reducedType() { - return SortField.Type.DOUBLE; - } - } - - private final SearchScript script; - - private final double[] values; - private double bottom; - - public DoubleScriptDataComparator(int numHits, SearchScript script) { - this.script = script; - values = new double[numHits]; - } - - @Override - public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - script.setNextReader(context); - return this; - } - - @Override - public void setScorer(Scorer scorer) { - script.setScorer(scorer); - } - - @Override - public int compare(int slot1, int slot2) { - final double v1 = values[slot1]; - final double v2 = values[slot2]; - if (v1 > v2) { - return 1; - } else if (v1 < v2) { - return -1; - } else { - return 0; - } - } - - @Override - public int compareBottom(int doc) { - script.setNextDocId(doc); - final double v2 = script.runAsDouble(); - if (bottom > v2) { - return 1; - } else if (bottom < v2) { - return -1; - } else { - return 0; - } - } - - @Override - public int compareTop(int doc) throws IOException { - script.setNextDocId(doc); - double docValue = script.runAsDouble(); - return Double.compare(top, docValue); - } - - @Override - public void copy(int slot, int doc) { - script.setNextDocId(doc); - values[slot] = script.runAsDouble(); - } - - @Override - public void setBottom(final int bottom) { - this.bottom = values[bottom]; - } - - @Override - public Double value(int slot) { - return values[slot]; - } - - @Override - public void add(int slot, int doc) { - script.setNextDocId(doc); - values[slot] += script.runAsDouble(); - } - - @Override - public void divide(int slot, int divisor) { - values[slot] /= divisor; - } - - @Override - public void missing(int slot) { - values[slot] = Double.MAX_VALUE; - } - - @Override - public int compareBottomMissing() { - return Double.compare(bottom, Double.MAX_VALUE); - } - - @Override - public int compareTopMissing() { - return Double.compare(top, Double.MAX_VALUE); - } -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java deleted file mode 100644 index cadb6fc4dd7..00000000000 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -/** - */ -public class DoubleValuesComparator extends DoubleValuesComparatorBase { - - private final double[] values; - - public DoubleValuesComparator(IndexNumericFieldData indexFieldData, double missingValue, int numHits, MultiValueMode sortMode) { - super(indexFieldData, missingValue, sortMode); - this.values = new double[numHits]; - } - - @Override - public int compare(int slot1, int slot2) { - final double v1 = values[slot1]; - final double v2 = values[slot2]; - return compare(v1, v2); - } - - @Override - public void setBottom(int slot) { - this.bottom = values[slot]; - } - - @Override - public void copy(int slot, int doc) throws IOException { - values[slot] = readerValues.get(doc); - } - - @Override - public Double value(int slot) { - return Double.valueOf(values[slot]); - } - - @Override - public void add(int slot, int doc) { - values[slot] += readerValues.get(doc); - } - - @Override - public void divide(int slot, int divisor) { - values[slot] /= divisor; - } - - @Override - public void missing(int slot) { - values[slot] = missingValue; - } -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorBase.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorBase.java deleted file mode 100644 index 7f648f849e2..00000000000 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorBase.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.search.FieldComparator; -import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.NumericDoubleValues; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -abstract class DoubleValuesComparatorBase extends NumberComparatorBase { - - protected final IndexNumericFieldData indexFieldData; - protected final double missingValue; - protected double bottom; - protected NumericDoubleValues readerValues; - protected final MultiValueMode sortMode; - - public DoubleValuesComparatorBase(IndexNumericFieldData indexFieldData, double missingValue, MultiValueMode sortMode) { - this.indexFieldData = indexFieldData; - this.missingValue = missingValue; - this.sortMode = sortMode; - } - - @Override - public final int compareBottom(int doc) throws IOException { - final double v2 = readerValues.get(doc); - return compare(bottom, v2); - } - - @Override - public int compareTop(int doc) throws IOException { - return compare(top.doubleValue(), readerValues.get(doc)); - } - - protected NumericDoubleValues getNumericDoubleValues(AtomicReaderContext context) { - SortedNumericDoubleValues readerValues = indexFieldData.load(context).getDoubleValues(); - return sortMode.select(readerValues, missingValue); - } - - @Override - public final FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - this.readerValues = getNumericDoubleValues(context); - return this; - } - - @Override - public int compareBottomMissing() { - return compare(bottom, missingValue); - } - - @Override - public int compareTopMissing() { - return compare(top.doubleValue(), missingValue); - } - - static final int compare(double left, double right) { - return Double.compare(left, right); - } -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java index 22870a461cf..321dc5b53d1 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java +++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java @@ -19,27 +19,36 @@ package org.elasticsearch.index.fielddata.fieldcomparator; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.search.FieldCache.Doubles; import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.Scorer; import org.apache.lucene.search.SortField; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.NumericDoubleValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.MultiValueMode; import java.io.IOException; /** + * Comparator source for double values. */ public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparatorSource { private final IndexNumericFieldData indexFieldData; private final Object missingValue; private final MultiValueMode sortMode; + private final Nested nested; - public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) { + public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { this.indexFieldData = indexFieldData; this.missingValue = missingValue; this.sortMode = sortMode; + this.nested = nested; } @Override @@ -47,11 +56,42 @@ public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparato return SortField.Type.DOUBLE; } + protected SortedNumericDoubleValues getValues(AtomicReaderContext context) { + return indexFieldData.load(context).getDoubleValues(); + } + + protected void setScorer(Scorer scorer) {} + @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - assert fieldname.equals(indexFieldData.getFieldNames().indexName()); + assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName()); final double dMissingValue = (Double) missingObject(missingValue, reversed); - return new DoubleValuesComparator(indexFieldData, dMissingValue, numHits, sortMode); + // NOTE: it's important to pass null as a missing value in the constructor so that + // the comparator doesn't check docsWithField since we replace missing values in select() + return new FieldComparator.DoubleComparator(numHits, null, null, null) { + @Override + protected Doubles getDoubleValues(AtomicReaderContext context, String field) throws IOException { + final SortedNumericDoubleValues values = getValues(context); + final NumericDoubleValues selectedValues; + if (nested == null) { + selectedValues = sortMode.select(values, dMissingValue); + } else { + final FixedBitSet rootDocs = nested.rootDocs(context); + final FixedBitSet innerDocs = nested.innerDocs(context); + selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc()); + } + return new Doubles() { + @Override + public double get(int docID) { + return selectedValues.get(docID); + } + }; + } + @Override + public void setScorer(Scorer scorer) { + DoubleValuesComparatorSource.this.setScorer(scorer); + } + }; } } diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparator.java deleted file mode 100644 index 54887ca3036..00000000000 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparator.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -/** - */ -public final class FloatValuesComparator extends DoubleValuesComparatorBase { - - private final float[] values; - - public FloatValuesComparator(IndexNumericFieldData indexFieldData, float missingValue, int numHits, MultiValueMode sortMode) { - super(indexFieldData, missingValue, sortMode); - assert indexFieldData.getNumericType().requiredBits() <= 32; - this.values = new float[numHits]; - } - - @Override - public int compare(int slot1, int slot2) { - final float v1 = values[slot1]; - final float v2 = values[slot2]; - return Float.compare(v1, v2); - } - - @Override - public void setBottom(int slot) { - this.bottom = values[slot]; - } - - @Override - public void copy(int slot, int doc) throws IOException { - values[slot] = (float) readerValues.get(doc); - } - - @Override - public Float value(int slot) { - return Float.valueOf(values[slot]); - } - - @Override - public void add(int slot, int doc) { - values[slot] += (float) readerValues.get(doc); - } - - @Override - public void divide(int slot, int divisor) { - values[slot] /= divisor; - } - - @Override - public void missing(int slot) { - values[slot] = (float) missingValue; - } -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java index 371a5a553e5..e663ee5e30b 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java +++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java @@ -18,27 +18,35 @@ */ package org.elasticsearch.index.fielddata.fieldcomparator; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.search.FieldCache.Floats; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.SortField; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.NumericDoubleValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.MultiValueMode; import java.io.IOException; /** + * Comparator source for float values. */ public class FloatValuesComparatorSource extends IndexFieldData.XFieldComparatorSource { private final IndexNumericFieldData indexFieldData; private final Object missingValue; private final MultiValueMode sortMode; + private final Nested nested; - public FloatValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) { + public FloatValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { this.indexFieldData = indexFieldData; this.missingValue = missingValue; this.sortMode = sortMode; + this.nested = nested; } @Override @@ -48,9 +56,30 @@ public class FloatValuesComparatorSource extends IndexFieldData.XFieldComparator @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - assert fieldname.equals(indexFieldData.getFieldNames().indexName()); + assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName()); final float dMissingValue = (Float) missingObject(missingValue, reversed); - return new FloatValuesComparator(indexFieldData, dMissingValue, numHits, sortMode); + // NOTE: it's important to pass null as a missing value in the constructor so that + // the comparator doesn't check docsWithField since we replace missing values in select() + return new FieldComparator.FloatComparator(numHits, null, null, null) { + @Override + protected Floats getFloatValues(AtomicReaderContext context, String field) throws IOException { + final SortedNumericDoubleValues values = indexFieldData.load(context).getDoubleValues(); + final NumericDoubleValues selectedValues; + if (nested == null) { + selectedValues = sortMode.select(values, dMissingValue); + } else { + final FixedBitSet rootDocs = nested.rootDocs(context); + final FixedBitSet innerDocs = nested.innerDocs(context); + selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc()); + } + return new Floats() { + @Override + public float get(int docID) { + return (float) selectedValues.get(docID); + } + }; + } + }; } } diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparator.java deleted file mode 100644 index 55b3ab969c3..00000000000 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparator.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -/** - */ -public final class LongValuesComparator extends LongValuesComparatorBase { - - private final long[] values; - - public LongValuesComparator(IndexNumericFieldData indexFieldData, long missingValue, int numHits, MultiValueMode sortMode) { - super(indexFieldData, missingValue, sortMode); - this.values = new long[numHits]; - assert indexFieldData.getNumericType().requiredBits() <= 64; - } - - @Override - public int compare(int slot1, int slot2) { - final long v1 = values[slot1]; - final long v2 = values[slot2]; - return Long.compare(v1, v2); - } - - @Override - public void setBottom(int slot) { - this.bottom = values[slot]; - } - - public void copy(int slot, int doc) throws IOException { - values[slot] = readerValues.get(doc); - } - - @Override - public Long value(int slot) { - return Long.valueOf(values[slot]); - } - - @Override - public void add(int slot, int doc) { - values[slot] += readerValues.get(doc); - } - - @Override - public void divide(int slot, int divisor) { - values[slot] /= divisor; - } - - @Override - public void missing(int slot) { - values[slot] = missingValue; - } -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorBase.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorBase.java deleted file mode 100644 index ec0f116aa9f..00000000000 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorBase.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.FieldComparator; -import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -abstract class LongValuesComparatorBase extends NumberComparatorBase { - - protected final IndexNumericFieldData indexFieldData; - protected final long missingValue; - protected long bottom; - protected NumericDocValues readerValues; - protected final MultiValueMode sortMode; - - - public LongValuesComparatorBase(IndexNumericFieldData indexFieldData, long missingValue, MultiValueMode sortMode) { - this.indexFieldData = indexFieldData; - this.missingValue = missingValue; - this.sortMode = sortMode; - } - - @Override - public final int compareBottom(int doc) throws IOException { - long v2 = readerValues.get(doc); - return Long.compare(bottom, v2); - } - - @Override - public int compareTop(int doc) throws IOException { - return Long.compare(top.longValue(), readerValues.get(doc)); - } - - @Override - public final FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - SortedNumericDocValues readerValues = indexFieldData.load(context).getLongValues(); - this.readerValues = sortMode.select(readerValues, missingValue); - return this; - } - - @Override - public int compareBottomMissing() { - return Long.compare(bottom, missingValue); - } - - @Override - public int compareTopMissing() { - return Long.compare(top.longValue(), missingValue); - } -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java index 7cffddd52e0..86d649bd276 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java +++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java @@ -18,8 +18,13 @@ */ package org.elasticsearch.index.fielddata.fieldcomparator; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.FieldCache.Longs; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.SortField; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; @@ -28,17 +33,20 @@ import org.elasticsearch.search.MultiValueMode; import java.io.IOException; /** + * Comparator source for long values. */ public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorSource { private final IndexNumericFieldData indexFieldData; private final Object missingValue; private final MultiValueMode sortMode; + private final Nested nested; - public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) { + public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { this.indexFieldData = indexFieldData; this.missingValue = missingValue; this.sortMode = sortMode; + this.nested = nested; } @Override @@ -48,9 +56,31 @@ public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorS @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - assert fieldname.equals(indexFieldData.getFieldNames().indexName()); + assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName()); - final long dMissingValue = (Long) missingObject(missingValue, reversed); - return new LongValuesComparator(indexFieldData, dMissingValue, numHits, sortMode); + final Long dMissingValue = (Long) missingObject(missingValue, reversed); + // NOTE: it's important to pass null as a missing value in the constructor so that + // the comparator doesn't check docsWithField since we replace missing values in select() + return new FieldComparator.LongComparator(numHits, null, null, null) { + @Override + protected Longs getLongValues(AtomicReaderContext context, String field) throws IOException { + final SortedNumericDocValues values = indexFieldData.load(context).getLongValues(); + final NumericDocValues selectedValues; + if (nested == null) { + selectedValues = sortMode.select(values, dMissingValue); + } else { + final FixedBitSet rootDocs = nested.rootDocs(context); + final FixedBitSet innerDocs = nested.innerDocs(context); + selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc()); + } + return new Longs() { + @Override + public long get(int docID) { + return selectedValues.get(docID); + } + }; + } + + }; } } diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NestedWrappableComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NestedWrappableComparator.java deleted file mode 100644 index 6a5bcc14fba..00000000000 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NestedWrappableComparator.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.apache.lucene.search.FieldComparator; - -/** Base comparator which allows for nested sorting. */ -public abstract class NestedWrappableComparator extends FieldComparator { - - /** - * Assigns the underlying missing value to the specified slot, if the actual implementation supports missing value. - * - * @param slot The slot to assign the the missing value to. - */ - public abstract void missing(int slot); - - /** - * Compares the missing value to the bottom. - * - * @return any N < 0 if the bottom value is not competitive with the missing value, any N > 0 if the - * bottom value is competitive with the missing value and 0 if they are equal. - */ - public abstract int compareBottomMissing(); - - /** - * Compares the missing value to the top. - * - * @return any N < 0 if the tope is not competitive with the missing value, any N > 0 if the top is competitive - * with the top and 0 if they are equal. - */ - public abstract int compareTopMissing(); -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NumberComparatorBase.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NumberComparatorBase.java deleted file mode 100644 index 5315b797d91..00000000000 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NumberComparatorBase.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata.fieldcomparator; - - -/** - * Base FieldComparator class for number fields. - */ -// This is right now only used for sorting number based fields inside nested objects -public abstract class NumberComparatorBase extends NestedWrappableComparator { - - protected T top; - /** - * Adds numeric value at the specified doc to the specified slot. - * - * @param slot The specified slot - * @param doc The specified doc - */ - public abstract void add(int slot, int doc); - - /** - * Divides the value at the specified slot with the specified divisor. - * - * @param slot The specified slot - * @param divisor The specified divisor - */ - public abstract void divide(int slot, int divisor); - - @Override - public void setTopValue(T top) { - this.top = top; - } - -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/StringScriptDataComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/StringScriptDataComparator.java deleted file mode 100644 index dd8965b2848..00000000000 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/StringScriptDataComparator.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SortField; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.script.SearchScript; - -import java.io.IOException; - -/** - * - */ -public class StringScriptDataComparator extends FieldComparator { - - private BytesRef top; - - public static IndexFieldData.XFieldComparatorSource comparatorSource(SearchScript script) { - return new InnerSource(script); - } - - private static class InnerSource extends IndexFieldData.XFieldComparatorSource { - - private final SearchScript script; - - private InnerSource(SearchScript script) { - this.script = script; - } - - @Override - public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - return new StringScriptDataComparator(numHits, script); - } - - @Override - public SortField.Type reducedType() { - return SortField.Type.STRING; - } - } - - private final SearchScript script; - - private BytesRef[] values; // TODO maybe we can preallocate or use a sentinel to prevent the conditionals in compare - - private BytesRef bottom; - - private final BytesRef spare = new BytesRef(); - - private int spareDoc = -1; - - public StringScriptDataComparator(int numHits, SearchScript script) { - this.script = script; - values = new BytesRef[numHits]; - } - - @Override - public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - script.setNextReader(context); - spareDoc = -1; - return this; - } - - @Override - public void setScorer(Scorer scorer) { - script.setScorer(scorer); - } - - @Override - public int compare(int slot1, int slot2) { - final BytesRef val1 = values[slot1]; - final BytesRef val2 = values[slot2]; - if (val1 == null) { - if (val2 == null) { - return 0; - } - return -1; - } else if (val2 == null) { - return 1; - } - - return val1.compareTo(val2); - } - - @Override - public int compareBottom(int doc) { - if (bottom == null) { - return -1; - } - setSpare(doc); - return bottom.compareTo(spare); - } - - @Override - public int compareTop(int doc) throws IOException { - script.setNextDocId(doc); - setSpare(doc); - return top.compareTo(spare); - } - - private void setSpare(int doc) { - if (spareDoc == doc) { - return; - } - script.setNextDocId(doc); - spare.copyChars(script.run().toString()); - spareDoc = doc; - } - - @Override - public void copy(int slot, int doc) { - setSpare(doc); - if (values[slot] == null) { - values[slot] = BytesRef.deepCopyOf(spare); - } else { - values[slot].copyBytes(spare); - } - } - - @Override - public void setBottom(final int bottom) { - this.bottom = values[bottom]; - } - - @Override - public void setTopValue(BytesRef top) { - this.top = top; - } - - @Override - public BytesRef value(int slot) { - return values[slot]; - } -} diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java index fc8bc62133b..1208055b898 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java +++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java @@ -25,10 +25,8 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.Index; -import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; -import org.elasticsearch.index.fielddata.FieldDataType; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.fielddata.*; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.search.MultiValueMode; @@ -74,7 +72,7 @@ public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponen } @Override - public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) { + public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { throw new UnsupportedOperationException("no global ordinals sorting yet"); } diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java index b1add916836..0cfd4419200 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java +++ b/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java @@ -28,10 +28,8 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; -import org.elasticsearch.index.fielddata.FieldDataType; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; +import org.elasticsearch.index.fielddata.*; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.mapper.FieldMapper.Names; import org.elasticsearch.search.MultiValueMode; @@ -80,7 +78,7 @@ abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData) this, missingValue, sortMode); + public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) { + return new BytesRefFieldComparatorSource((IndexFieldData) this, missingValue, sortMode, nested); } @Override diff --git a/src/main/java/org/elasticsearch/index/search/nested/NestedFieldComparatorSource.java b/src/main/java/org/elasticsearch/index/search/nested/NestedFieldComparatorSource.java deleted file mode 100644 index bf462b709b1..00000000000 --- a/src/main/java/org/elasticsearch/index/search/nested/NestedFieldComparatorSource.java +++ /dev/null @@ -1,500 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.search.nested; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.SortField; -import org.apache.lucene.util.FixedBitSet; -import org.elasticsearch.ElasticsearchIllegalArgumentException; -import org.elasticsearch.common.lucene.docset.DocIdSets; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.fieldcomparator.NestedWrappableComparator; -import org.elasticsearch.index.fielddata.fieldcomparator.NumberComparatorBase; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; -import java.util.Locale; - -/** - */ -public class NestedFieldComparatorSource extends IndexFieldData.XFieldComparatorSource { - - private final MultiValueMode sortMode; - private final IndexFieldData.XFieldComparatorSource wrappedSource; - private final Filter rootDocumentsFilter; - private final Filter innerDocumentsFilter; - - public NestedFieldComparatorSource(MultiValueMode sortMode, IndexFieldData.XFieldComparatorSource wrappedSource, Filter rootDocumentsFilter, Filter innerDocumentsFilter) { - this.sortMode = sortMode; - this.wrappedSource = wrappedSource; - this.rootDocumentsFilter = rootDocumentsFilter; - this.innerDocumentsFilter = innerDocumentsFilter; - } - - @Override - public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - // +1: have one spare slot for value comparison between inner documents. - FieldComparator wrappedComparator = wrappedSource.newComparator(fieldname, numHits + 1, sortPos, reversed); - switch (sortMode) { - case MAX: - return new NestedFieldComparator.Highest(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits); - case MIN: - return new NestedFieldComparator.Lowest(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits); - case SUM: - return new NestedFieldComparator.Sum((NumberComparatorBase) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits); - case AVG: - return new NestedFieldComparator.Avg((NumberComparatorBase) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits); - default: - throw new ElasticsearchIllegalArgumentException( - String.format(Locale.ROOT, "Unsupported sort_mode[%s] for nested type", sortMode) - ); - } - } - - @Override - public SortField.Type reducedType() { - return wrappedSource.reducedType(); - } - -} - -abstract class NestedFieldComparator extends FieldComparator { - - final Filter rootDocumentsFilter; - final Filter innerDocumentsFilter; - final int spareSlot; - - FieldComparator wrappedComparator; - FixedBitSet rootDocuments; - FixedBitSet innerDocuments; - int bottomSlot; - Object top; - - NestedFieldComparator(FieldComparator wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) { - this.wrappedComparator = wrappedComparator; - this.rootDocumentsFilter = rootDocumentsFilter; - this.innerDocumentsFilter = innerDocumentsFilter; - this.spareSlot = spareSlot; - } - - @Override - public final int compare(int slot1, int slot2) { - return wrappedComparator.compare(slot1, slot2); - } - - @Override - public final void setBottom(int slot) { - wrappedComparator.setBottom(slot); - this.bottomSlot = slot; - } - - @Override - public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - DocIdSet innerDocuments = innerDocumentsFilter.getDocIdSet(context, null); - if (DocIdSets.isEmpty(innerDocuments)) { - this.innerDocuments = null; - } else if (innerDocuments instanceof FixedBitSet) { - this.innerDocuments = (FixedBitSet) innerDocuments; - } else { - this.innerDocuments = DocIdSets.toFixedBitSet(innerDocuments.iterator(), context.reader().maxDoc()); - } - DocIdSet rootDocuments = rootDocumentsFilter.getDocIdSet(context, null); - if (DocIdSets.isEmpty(rootDocuments)) { - this.rootDocuments = null; - } else if (rootDocuments instanceof FixedBitSet) { - this.rootDocuments = (FixedBitSet) rootDocuments; - } else { - this.rootDocuments = DocIdSets.toFixedBitSet(rootDocuments.iterator(), context.reader().maxDoc()); - } - - wrappedComparator = wrappedComparator.setNextReader(context); - return this; - } - - @Override - public final Object value(int slot) { - return wrappedComparator.value(slot); - } - - @Override - public void setTopValue(Object top) { - this.top = top; - wrappedComparator.setTopValue(top); - } - - final static class Lowest extends NestedFieldComparator { - - Lowest(FieldComparator wrappedComparator, Filter parentFilter, Filter childFilter, int spareSlot) { - super(wrappedComparator, parentFilter, childFilter, spareSlot); - } - - @Override - public int compareBottom(int rootDoc) throws IOException { - if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { - return compareBottomMissing(wrappedComparator); - } - - // We need to copy the lowest value from all nested docs into slot. - int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); - int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return compareBottomMissing(wrappedComparator); - } - - // We only need to emit a single cmp value for any matching nested doc - int cmp = wrappedComparator.compareBottom(nestedDoc); - if (cmp > 0) { - return cmp; - } - - while (true) { - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return cmp; - } - int cmp1 = wrappedComparator.compareBottom(nestedDoc); - if (cmp1 > 0) { - return cmp1; - } else { - if (cmp1 == 0) { - cmp = 0; - } - } - } - } - - @Override - public void copy(int slot, int rootDoc) throws IOException { - if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { - copyMissing(wrappedComparator, slot); - return; - } - - // We need to copy the lowest value from all nested docs into slot. - int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); - int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - copyMissing(wrappedComparator, slot); - return; - } - wrappedComparator.copy(slot, nestedDoc); - - while (true) { - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return; - } - wrappedComparator.copy(spareSlot, nestedDoc); - if (wrappedComparator.compare(spareSlot, slot) < 0) { - wrappedComparator.copy(slot, nestedDoc); - } - } - } - - @Override - public int compareTop(int rootDoc) throws IOException { - if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { - return compareTopMissing(wrappedComparator); - } - - // We need to copy the lowest value from all nested docs into slot. - int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); - int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return compareTopMissing(wrappedComparator); - } - - // We only need to emit a single cmp value for any matching nested doc - @SuppressWarnings("unchecked") - int cmp = wrappedComparator.compareTop(nestedDoc); - if (cmp > 0) { - return cmp; - } - - while (true) { - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return cmp; - } - @SuppressWarnings("unchecked") - int cmp1 = wrappedComparator.compareTop(nestedDoc); - if (cmp1 > 0) { - return cmp1; - } else { - if (cmp1 == 0) { - cmp = 0; - } - } - } - } - } - - final static class Highest extends NestedFieldComparator { - - Highest(FieldComparator wrappedComparator, Filter parentFilter, Filter childFilter, int spareSlot) { - super(wrappedComparator, parentFilter, childFilter, spareSlot); - } - - @Override - public int compareBottom(int rootDoc) throws IOException { - if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { - return compareBottomMissing(wrappedComparator); - } - - int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); - int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return compareBottomMissing(wrappedComparator); - } - - int cmp = wrappedComparator.compareBottom(nestedDoc); - if (cmp < 0) { - return cmp; - } - - while (true) { - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return cmp; - } - int cmp1 = wrappedComparator.compareBottom(nestedDoc); - if (cmp1 < 0) { - return cmp1; - } else if (cmp1 == 0) { - cmp = 0; - } - } - } - - @Override - public void copy(int slot, int rootDoc) throws IOException { - if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { - copyMissing(wrappedComparator, slot); - return; - } - - int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); - int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - copyMissing(wrappedComparator, slot); - return; - } - wrappedComparator.copy(slot, nestedDoc); - - while (true) { - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return; - } - wrappedComparator.copy(spareSlot, nestedDoc); - if (wrappedComparator.compare(spareSlot, slot) > 0) { - wrappedComparator.copy(slot, nestedDoc); - } - } - } - - @Override - public int compareTop(int rootDoc) throws IOException { - if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { - return compareTopMissing(wrappedComparator); - } - - int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); - int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return compareTopMissing(wrappedComparator); - } - - @SuppressWarnings("unchecked") - int cmp = wrappedComparator.compareTop(nestedDoc); - if (cmp < 0) { - return cmp; - } - - while (true) { - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return cmp; - } - @SuppressWarnings("unchecked") - int cmp1 = wrappedComparator.compareTop(nestedDoc); - if (cmp1 < 0) { - return cmp1; - } else if (cmp1 == 0) { - cmp = 0; - } - } - } - } - - static abstract class NumericNestedFieldComparatorBase extends NestedFieldComparator { - protected NumberComparatorBase numberComparator; - - NumericNestedFieldComparatorBase(NumberComparatorBase wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) { - super(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, spareSlot); - this.numberComparator = wrappedComparator; - } - - @Override - public final int compareBottom(int rootDoc) throws IOException { - if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { - return compareBottomMissing(wrappedComparator); - } - - final int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); - int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return compareBottomMissing(wrappedComparator); - } - - int counter = 1; - wrappedComparator.copy(spareSlot, nestedDoc); - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - while (nestedDoc > prevRootDoc && nestedDoc < rootDoc) { - onNested(spareSlot, nestedDoc); - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - counter++; - } - afterNested(spareSlot, counter); - return compare(bottomSlot, spareSlot); - } - - @Override - public final void copy(int slot, int rootDoc) throws IOException { - if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { - copyMissing(wrappedComparator, slot); - return; - } - - final int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); - int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - copyMissing(wrappedComparator, slot); - return; - } - int counter = 1; - wrappedComparator.copy(slot, nestedDoc); - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - while (nestedDoc > prevRootDoc && nestedDoc < rootDoc) { - onNested(slot, nestedDoc); - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - counter++; - } - afterNested(slot, counter); - } - - @Override - @SuppressWarnings("unchecked") - public int compareTop(int rootDoc) throws IOException { - if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) { - return compareTopMissing(wrappedComparator); - } - - final int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1); - int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1); - if (nestedDoc >= rootDoc || nestedDoc == -1) { - return compareTopMissing(wrappedComparator); - } - - int counter = 1; - wrappedComparator.copy(spareSlot, nestedDoc); - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - while (nestedDoc > prevRootDoc && nestedDoc < rootDoc) { - onNested(spareSlot, nestedDoc); - nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1); - counter++; - } - afterNested(spareSlot, counter); - return wrappedComparator.compareValues(wrappedComparator.value(spareSlot), top); - } - - protected abstract void onNested(int slot, int nestedDoc); - - protected abstract void afterNested(int slot, int count); - - @Override - public final FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - super.setNextReader(context); - numberComparator = (NumberComparatorBase) super.wrappedComparator; - return this; - } - } - - final static class Sum extends NumericNestedFieldComparatorBase { - - Sum(NumberComparatorBase wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) { - super(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, spareSlot); - } - - @Override - protected void onNested(int slot, int nestedDoc) { - numberComparator.add(slot, nestedDoc); - } - - @Override - protected void afterNested(int slot, int count) { - } - - } - - final static class Avg extends NumericNestedFieldComparatorBase { - Avg(NumberComparatorBase wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) { - super(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, spareSlot); - } - - @Override - protected void onNested(int slot, int nestedDoc) { - numberComparator.add(slot, nestedDoc); - } - - @Override - protected void afterNested(int slot, int count) { - numberComparator.divide(slot, count); - } - - - } - - static final void copyMissing(FieldComparator comparator, int slot) { - if (comparator instanceof NestedWrappableComparator) { - ((NestedWrappableComparator) comparator).missing(slot); - } - } - - static final int compareBottomMissing(FieldComparator comparator) { - if (comparator instanceof NestedWrappableComparator) { - return ((NestedWrappableComparator) comparator).compareBottomMissing(); - } else { - return 0; - } - } - - @SuppressWarnings("unchecked") - static final int compareTopMissing(FieldComparator comparator) { - if (comparator instanceof NestedWrappableComparator) { - return ((NestedWrappableComparator) comparator).compareTopMissing(); - } else { - return 0; - } - } - -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/search/MultiValueMode.java b/src/main/java/org/elasticsearch/search/MultiValueMode.java index be741da8bfa..f87b54ba0fe 100644 --- a/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -23,6 +23,7 @@ package org.elasticsearch.search; import org.apache.lucene.index.*; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.NumericDoubleValues; @@ -91,7 +92,13 @@ public enum MultiValueMode { */ @Override public long reduce(long a, int numValues) { - return Math.round(a / Math.max(1.0, numValues)); + if (numValues <= 1) { + // without this the average might be different from the original + // values on a single-valued field due to the precision loss of the double + return a; + } else { + return Math.round(a / Math.max(1.0, numValues)); + } } }, @@ -115,6 +122,16 @@ public enum MultiValueMode { return Math.min(a, b); } + @Override + public int applyOrd(int ord1, int ord2) { + return Math.min(ord1, ord2); + } + + @Override + public BytesRef apply(BytesRef a, BytesRef b) { + return a.compareTo(b) <= 0 ? a : b; + } + /** * Returns {@link Double#POSITIVE_INFINITY} */ @@ -159,11 +176,11 @@ public enum MultiValueMode { } @Override - protected int pick(RandomAccessOrds values, int missingOrd) { + protected int pick(RandomAccessOrds values) { if (values.cardinality() > 0) { return (int) values.ordAt(0); } else { - return missingOrd; + return -1; } } }, @@ -188,6 +205,16 @@ public enum MultiValueMode { return Math.max(a, b); } + @Override + public int applyOrd(int ord1, int ord2) { + return Math.max(ord1, ord2); + } + + @Override + public BytesRef apply(BytesRef a, BytesRef b) { + return a.compareTo(b) > 0 ? a : b; + } + /** * Returns {@link Double#NEGATIVE_INFINITY} */ @@ -236,12 +263,12 @@ public enum MultiValueMode { } @Override - protected int pick(RandomAccessOrds values, int missingOrd) { + protected int pick(RandomAccessOrds values) { final int count = values.cardinality(); if (count > 0) { return (int) values.ordAt(count - 1); } else { - return missingOrd; + return -1; } } }; @@ -274,6 +301,14 @@ public enum MultiValueMode { */ public abstract long apply(long a, long b); + public int applyOrd(int ord1, int ord2) { + throw new UnsupportedOperationException(); + } + + public BytesRef apply(BytesRef a, BytesRef b) { + throw new UnsupportedOperationException(); + } + /** * Returns an initial value for the sort mode that is guaranteed to have no impact if passed * to {@link #apply(double, double)}. This value should be used as the initial value if the @@ -359,6 +394,11 @@ public enum MultiValueMode { } } + /** + * Return a {@link NumericDocValues} instance that can be used to sort documents + * with this mode and the provided values. When a document has no value, + * missingValue is returned. + */ public NumericDocValues select(final SortedNumericDocValues values, final long missingValue) { final NumericDocValues singleton = DocValues.unwrapSingleton(values); if (singleton != null) { @@ -388,6 +428,51 @@ public enum MultiValueMode { } } + /** + * Return a {@link NumericDocValues} instance that can be used to sort root documents + * with this mode, the provided values and filters for root/inner documents. + * + * For every root document, the values of its inner documents will be aggregated. + * If none of the inner documents has a value, then missingValue is returned. + * + * NOTE: Calling the returned instance on docs that are not root docs is illegal + */ + public NumericDocValues select(final SortedNumericDocValues values, final long missingValue, final FixedBitSet rootDocs, final FixedBitSet innerDocs, int maxDoc) { + if (rootDocs == null || innerDocs == null) { + return select(FieldData.emptySortedNumeric(maxDoc), missingValue); + } + return new NumericDocValues() { + + @Override + public long get(int rootDoc) { + assert rootDocs.get(rootDoc) : "can only sort root documents"; + if (rootDoc == 0) { + return missingValue; + } + + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1); + + long accumulated = startLong(); + int numValues = 0; + + for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) { + values.setDocument(doc); + final int count = values.count(); + for (int i = 0; i < count; ++i) { + final long value = values.valueAt(i); + accumulated = apply(accumulated, value); + } + numValues += count; + } + + return numValues == 0 + ? missingValue + : reduce(accumulated, numValues); + } + }; + } + protected double pick(SortedNumericDoubleValues values, double missingValue) { final int count = values.count(); if (count == 0) { @@ -401,6 +486,11 @@ public enum MultiValueMode { } } + /** + * Return a {@link NumericDoubleValues} instance that can be used to sort documents + * with this mode and the provided values. When a document has no value, + * missingValue is returned. + */ public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue) { final NumericDoubleValues singleton = FieldData.unwrapSingleton(values); if (singleton != null) { @@ -430,15 +520,65 @@ public enum MultiValueMode { } } + /** + * Return a {@link NumericDoubleValues} instance that can be used to sort root documents + * with this mode, the provided values and filters for root/inner documents. + * + * For every root document, the values of its inner documents will be aggregated. + * If none of the inner documents has a value, then missingValue is returned. + * + * NOTE: Calling the returned instance on docs that are not root docs is illegal + */ + public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final FixedBitSet rootDocs, final FixedBitSet innerDocs, int maxDoc) { + if (rootDocs == null || innerDocs == null) { + return select(FieldData.emptySortedNumericDoubles(maxDoc), missingValue); + } + return new NumericDoubleValues() { + + @Override + public double get(int rootDoc) { + assert rootDocs.get(rootDoc) : "can only sort root documents"; + if (rootDoc == 0) { + return missingValue; + } + + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1); + + double accumulated = startDouble(); + int numValues = 0; + + for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) { + values.setDocument(doc); + final int count = values.count(); + for (int i = 0; i < count; ++i) { + final double value = values.valueAt(i); + accumulated = apply(accumulated, value); + } + numValues += count; + } + + return numValues == 0 + ? missingValue + : reduce(accumulated, numValues); + } + }; + } + protected BytesRef pick(SortedBinaryDocValues values, BytesRef missingValue) { throw new ElasticsearchIllegalArgumentException("Unsupported sort mode: " + this); } + /** + * Return a {@link BinaryDocValues} instance that can be used to sort documents + * with this mode and the provided values. When a document has no value, + * missingValue is returned. + */ public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue) { final BinaryDocValues singleton = FieldData.unwrapSingleton(values); if (singleton != null) { final Bits docsWithField = FieldData.unwrapSingletonBits(values); - if (docsWithField == null || missingValue == null || missingValue.length == 0) { + if (docsWithField == null) { return singleton; } else { return new BinaryDocValues() { @@ -463,47 +603,85 @@ public enum MultiValueMode { } } - protected int pick(RandomAccessOrds values, int missingOrd) { + /** + * Return a {@link BinaryDocValues} instance that can be used to sort root documents + * with this mode, the provided values and filters for root/inner documents. + * + * For every root document, the values of its inner documents will be aggregated. + * If none of the inner documents has a value, then missingValue is returned. + * + * NOTE: Calling the returned instance on docs that are not root docs is illegal + */ + public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final FixedBitSet rootDocs, final FixedBitSet innerDocs, int maxDoc) { + if (rootDocs == null || innerDocs == null) { + return select(FieldData.emptySortedBinary(maxDoc), missingValue); + } + final BinaryDocValues selectedValues = select(values, new BytesRef()); + final Bits docsWithValue; + if (FieldData.unwrapSingleton(values) != null) { + docsWithValue = FieldData.unwrapSingletonBits(values); + } else { + docsWithValue = FieldData.docsWithValue(values, maxDoc); + } + return new BinaryDocValues() { + + final BytesRef spare = new BytesRef(); + + @Override + public BytesRef get(int rootDoc) { + assert rootDocs.get(rootDoc) : "can only sort root documents"; + if (rootDoc == 0) { + return missingValue; + } + + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1); + + BytesRef accumulated = null; + + for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) { + values.setDocument(doc); + final BytesRef innerValue = selectedValues.get(doc); + if (innerValue.length > 0 || docsWithValue == null || docsWithValue.get(doc)) { + if (accumulated == null) { + spare.copyBytes(innerValue); + accumulated = spare; + } else { + final BytesRef applied = apply(accumulated, innerValue); + if (applied == innerValue) { + accumulated.copyBytes(innerValue); + } + } + } + } + + return accumulated == null ? missingValue : accumulated; + } + }; + } + + protected int pick(RandomAccessOrds values) { throw new ElasticsearchIllegalArgumentException("Unsupported sort mode: " + this); } - public SortedDocValues select(final RandomAccessOrds values, final int missingOrd) { + /** + * Return a {@link SortedDocValues} instance that can be used to sort documents + * with this mode and the provided values. + */ + public SortedDocValues select(final RandomAccessOrds values) { if (values.getValueCount() >= Integer.MAX_VALUE) { throw new UnsupportedOperationException("fields containing more than " + (Integer.MAX_VALUE-1) + " unique terms are unsupported"); } final SortedDocValues singleton = DocValues.unwrapSingleton(values); if (singleton != null) { - if (missingOrd == -1) { - return singleton; - } else { - return new SortedDocValues() { - @Override - public int getOrd(int docID) { - final int ord = singleton.getOrd(docID); - if (ord == -1) { - return missingOrd; - } - return ord; - } - - @Override - public BytesRef lookupOrd(int ord) { - return values.lookupOrd(ord); - } - - @Override - public int getValueCount() { - return (int) values.getValueCount(); - } - }; - } + return singleton; } else { return new SortedDocValues() { @Override public int getOrd(int docID) { values.setDocument(docID); - return pick(values, missingOrd); + return pick(values); } @Override @@ -519,4 +697,55 @@ public enum MultiValueMode { } } + /** + * Return a {@link SortedDocValues} instance that can be used to sort root documents + * with this mode, the provided values and filters for root/inner documents. + * + * For every root document, the values of its inner documents will be aggregated. + * + * NOTE: Calling the returned instance on docs that are not root docs is illegal + */ + public SortedDocValues select(final RandomAccessOrds values, final FixedBitSet rootDocs, final FixedBitSet innerDocs) { + if (rootDocs == null || innerDocs == null) { + return select((RandomAccessOrds) DocValues.emptySortedSet()); + } + final SortedDocValues selectedValues = select(values); + return new SortedDocValues() { + + @Override + public BytesRef lookupOrd(int ord) { + return selectedValues.lookupOrd(ord); + } + + @Override + public int getValueCount() { + return selectedValues.getValueCount(); + } + + @Override + public int getOrd(int rootDoc) { + assert rootDocs.get(rootDoc) : "can only sort root documents"; + if (rootDoc == 0) { + return -1; + } + + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1); + int ord = -1; + + for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) { + final int innerOrd = selectedValues.getOrd(doc); + if (innerOrd != -1) { + if (ord == -1) { + ord = innerOrd; + } else { + ord = applyOrd(ord, innerOrd); + } + } + } + + return ord; + } + }; + } } diff --git a/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java b/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java index 97b815eea76..9832888d5c9 100644 --- a/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java +++ b/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java @@ -20,10 +20,11 @@ package org.elasticsearch.search.sort; import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.search.FieldCache.Doubles; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.Filter; import org.apache.lucene.search.SortField; -import org.apache.lucene.search.SortField.Type; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance; @@ -32,12 +33,11 @@ import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.*; -import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparator; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.ObjectMappers; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.query.ParsedFilter; -import org.elasticsearch.index.search.nested.NestedFieldComparatorSource; import org.elasticsearch.index.search.nested.NonNestedDocsFilter; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.internal.SearchContext; @@ -127,27 +127,6 @@ public class GeoDistanceSortParser implements SortParser { final MultiValueMode finalSortMode = sortMode; // final reference for use in the anonymous class final IndexGeoPointFieldData geoIndexFieldData = context.fieldData().getForField(mapper); final FixedSourceDistance distance = geoDistance.fixedSourceDistance(point.lat(), point.lon(), unit); - IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() { - - @Override - public Type reducedType() { - return Type.DOUBLE; - } - - @Override - public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { - return new DoubleValuesComparator(null, Double.MAX_VALUE, numHits, finalSortMode) { - @Override - protected NumericDoubleValues getNumericDoubleValues(AtomicReaderContext context) { - final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues(); - final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(distance, geoPointValues); - return sortMode.select(distanceValues, Double.MAX_VALUE); - } - - }; - } - - }; ObjectMapper objectMapper; if (nestedPath != null) { ObjectMappers objectMappers = context.mapperService().objectMapper(nestedPath); @@ -161,6 +140,7 @@ public class GeoDistanceSortParser implements SortParser { } else { objectMapper = context.mapperService().resolveClosestNestedObjectMapper(fieldName); } + final Nested nested; if (objectMapper != null && objectMapper.nested().isNested()) { Filter rootDocumentsFilter = context.filterCache().cache(NonNestedDocsFilter.INSTANCE); Filter innerDocumentsFilter; @@ -169,11 +149,45 @@ public class GeoDistanceSortParser implements SortParser { } else { innerDocumentsFilter = context.filterCache().cache(objectMapper.nestedTypeFilter()); } - geoDistanceComparatorSource = new NestedFieldComparatorSource( - sortMode, geoDistanceComparatorSource, rootDocumentsFilter, innerDocumentsFilter - ); + nested = new Nested(rootDocumentsFilter, innerDocumentsFilter); + } else { + nested = null; } + IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() { + + @Override + public SortField.Type reducedType() { + return SortField.Type.DOUBLE; + } + + @Override + public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new FieldComparator.DoubleComparator(numHits, null, null, null) { + @Override + protected Doubles getDoubleValues(AtomicReaderContext context, String field) throws IOException { + final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues(); + final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(distance, geoPointValues); + final NumericDoubleValues selectedValues; + if (nested == null) { + selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE); + } else { + final FixedBitSet rootDocs = nested.rootDocs(context); + final FixedBitSet innerDocs = nested.innerDocs(context); + selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE, rootDocs, innerDocs, context.reader().maxDoc()); + } + return new Doubles() { + @Override + public double get(int docID) { + return selectedValues.get(docID); + } + }; + } + }; + } + + }; + return new SortField(fieldName, geoDistanceComparatorSource, reverse); } } diff --git a/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java b/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java index b368628cc2d..861a2ceac22 100644 --- a/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java +++ b/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java @@ -19,21 +19,25 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Scorer; import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.fieldcomparator.DoubleScriptDataComparator; -import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.index.fielddata.fieldcomparator.StringScriptDataComparator; +import org.elasticsearch.index.fielddata.*; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; import org.elasticsearch.index.mapper.ObjectMappers; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.query.ParsedFilter; -import org.elasticsearch.index.search.nested.NestedFieldComparatorSource; import org.elasticsearch.index.search.nested.NonNestedDocsFilter; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.internal.SearchContext; @@ -44,6 +48,9 @@ import java.util.Map; */ public class ScriptSortParser implements SortParser { + private static final String STRING_SORT_TYPE = "string"; + private static final String NUMBER_SORT_TYPE = "number"; + @Override public String[] names() { return new String[]{"_script"}; @@ -105,17 +112,9 @@ public class ScriptSortParser implements SortParser { if (type == null) { throw new SearchParseException(context, "_script sorting requires setting the type of the script"); } - SearchScript searchScript = context.scriptService().search(context.lookup(), scriptLang, script, scriptType, params); - IndexFieldData.XFieldComparatorSource fieldComparatorSource; - if ("string".equals(type)) { - fieldComparatorSource = StringScriptDataComparator.comparatorSource(searchScript); - } else if ("number".equals(type)) { - fieldComparatorSource = DoubleScriptDataComparator.comparatorSource(searchScript); - } else { - throw new SearchParseException(context, "custom script sort type [" + type + "] not supported"); - } + final SearchScript searchScript = context.scriptService().search(context.lookup(), scriptLang, script, scriptType, params); - if ("string".equals(type) && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) { + if (STRING_SORT_TYPE.equals(type) && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) { throw new SearchParseException(context, "type [string] doesn't support mode [" + sortMode + "]"); } @@ -125,6 +124,7 @@ public class ScriptSortParser implements SortParser { // If nested_path is specified, then wrap the `fieldComparatorSource` in a `NestedFieldComparatorSource` ObjectMapper objectMapper; + final Nested nested; if (nestedPath != null) { ObjectMappers objectMappers = context.mapperService().objectMapper(nestedPath); if (objectMappers == null) { @@ -142,7 +142,58 @@ public class ScriptSortParser implements SortParser { } else { innerDocumentsFilter = context.filterCache().cache(objectMapper.nestedTypeFilter()); } - fieldComparatorSource = new NestedFieldComparatorSource(sortMode, fieldComparatorSource, rootDocumentsFilter, innerDocumentsFilter); + nested = new Nested(rootDocumentsFilter, innerDocumentsFilter); + } else { + nested = null; + } + + final IndexFieldData.XFieldComparatorSource fieldComparatorSource; + switch (type) { + case STRING_SORT_TYPE: + fieldComparatorSource = new BytesRefFieldComparatorSource(null, null, sortMode, nested) { + @Override + protected SortedBinaryDocValues getValues(AtomicReaderContext context) { + searchScript.setNextReader(context); + final BinaryDocValues values = new BinaryDocValues() { + final BytesRef spare = new BytesRef(); + @Override + public BytesRef get(int docID) { + searchScript.setNextDocId(docID); + spare.copyChars(searchScript.run().toString()); + return spare; + } + }; + return FieldData.singleton(values, null); + } + @Override + protected void setScorer(Scorer scorer) { + searchScript.setScorer(scorer); + } + }; + break; + case NUMBER_SORT_TYPE: + // TODO: should we rather sort missing values last? + fieldComparatorSource = new DoubleValuesComparatorSource(null, Double.MAX_VALUE, sortMode, nested) { + @Override + protected SortedNumericDoubleValues getValues(AtomicReaderContext context) { + searchScript.setNextReader(context); + final NumericDoubleValues values = new NumericDoubleValues() { + @Override + public double get(int docID) { + searchScript.setNextDocId(docID); + return searchScript.runAsDouble(); + } + }; + return FieldData.singleton(values, null); + } + @Override + protected void setScorer(Scorer scorer) { + searchScript.setScorer(scorer); + } + }; + break; + default: + throw new SearchParseException(context, "custom script sort type [" + type + "] not supported"); } return new SortField("_script", fieldComparatorSource, reverse); diff --git a/src/main/java/org/elasticsearch/search/sort/SortParseElement.java b/src/main/java/org/elasticsearch/search/sort/SortParseElement.java index d4e27969ced..1f1a3a147b5 100644 --- a/src/main/java/org/elasticsearch/search/sort/SortParseElement.java +++ b/src/main/java/org/elasticsearch/search/sort/SortParseElement.java @@ -28,14 +28,14 @@ import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.ObjectMappers; import org.elasticsearch.index.mapper.core.NumberFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.query.ParsedFilter; -import org.elasticsearch.index.search.nested.NestedFieldComparatorSource; import org.elasticsearch.index.search.nested.NonNestedDocsFilter; +import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.internal.SearchContext; @@ -190,7 +190,7 @@ public class SortParseElement implements SearchParseElement { sortFields.add(SORT_DOC); } } else { - FieldMapper fieldMapper = context.smartNameFieldMapper(fieldName); + FieldMapper fieldMapper = context.smartNameFieldMapper(fieldName); if (fieldMapper == null) { if (ignoreUnmapped) { return; @@ -219,8 +219,7 @@ public class SortParseElement implements SearchParseElement { sortMode = resolveDefaultSortMode(reverse); } - IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.fieldData().getForField(fieldMapper) - .comparatorSource(missing, sortMode); + ObjectMapper objectMapper; if (nestedPath != null) { ObjectMappers objectMappers = context.mapperService().objectMapper(nestedPath); @@ -234,6 +233,7 @@ public class SortParseElement implements SearchParseElement { } else { objectMapper = context.mapperService().resolveClosestNestedObjectMapper(fieldName); } + final Nested nested; if (objectMapper != null && objectMapper.nested().isNested()) { Filter rootDocumentsFilter = context.filterCache().cache(NonNestedDocsFilter.INSTANCE); Filter innerDocumentsFilter; @@ -242,8 +242,13 @@ public class SortParseElement implements SearchParseElement { } else { innerDocumentsFilter = context.filterCache().cache(objectMapper.nestedTypeFilter()); } - fieldComparatorSource = new NestedFieldComparatorSource(sortMode, fieldComparatorSource, rootDocumentsFilter, innerDocumentsFilter); + nested = new Nested(rootDocumentsFilter, innerDocumentsFilter); + } else { + nested = null; } + + IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.fieldData().getForField(fieldMapper) + .comparatorSource(missing, sortMode, nested); sortFields.add(new SortField(fieldMapper.names().indexName(), fieldComparatorSource, reverse)); } } diff --git a/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java b/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java index 308fb8df9fd..0fc64eed58d 100644 --- a/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java +++ b/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java @@ -28,17 +28,18 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.test.ElasticsearchTestCase; +import org.elasticsearch.test.ElasticsearchTestCase.UsesLuceneFieldCacheOnPurpose; import org.junit.Test; import java.io.IOException; import java.util.ArrayList; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; /** * */ +@UsesLuceneFieldCacheOnPurpose public class SimpleLuceneTests extends ElasticsearchTestCase { @Test diff --git a/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTests.java b/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTests.java index d6ebea38e7b..82ffdaf7e9a 100644 --- a/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTests.java +++ b/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTests.java @@ -24,7 +24,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.search.*; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; -import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; import org.elasticsearch.search.MultiValueMode; import org.junit.Test; @@ -100,7 +100,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests TopFieldDocs topDocs; topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN)))); + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one())); @@ -110,7 +110,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three())); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true))); + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(2)); assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); @@ -167,14 +167,14 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests assertValues(bytesValues, 2, three()); IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); - TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN)))); + TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs.length, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); - topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true))); + topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs.length, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); @@ -226,7 +226,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN)))); + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(7)); @@ -242,12 +242,12 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests assertThat(topDocs.scoreDocs[5].doc, equalTo(6)); assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("08")); assertThat(topDocs.scoreDocs[6].doc, equalTo(1)); - assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(BytesRefFieldComparatorSource.MAX_TERM)); + assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(XFieldComparatorSource.MAX_TERM)); assertThat(topDocs.scoreDocs[7].doc, equalTo(5)); - assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(BytesRefFieldComparatorSource.MAX_TERM)); + assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(XFieldComparatorSource.MAX_TERM)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true))); + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(6)); diff --git a/src/test/java/org/elasticsearch/index/fielddata/AbstractNumericFieldDataTests.java b/src/test/java/org/elasticsearch/index/fielddata/AbstractNumericFieldDataTests.java index 3ede266ac43..97f46e3ff30 100644 --- a/src/test/java/org/elasticsearch/index/fielddata/AbstractNumericFieldDataTests.java +++ b/src/test/java/org/elasticsearch/index/fielddata/AbstractNumericFieldDataTests.java @@ -91,14 +91,14 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp TopFieldDocs topDocs; topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN)))); + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true))); + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(2)); assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); @@ -145,42 +145,42 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp TopFieldDocs topDocs; topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN)))); // defaults to _last + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); // defaults to _last assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true))); // defaults to _last + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); // defaults to _last assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(2)); assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN)))); + new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN, null)))); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX), true))); + new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX, null), true))); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); assertThat(topDocs.scoreDocs[2].doc, equalTo(0)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MIN)))); + new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MIN, null)))); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MAX), true))); + new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MAX, null), true))); assertThat(topDocs.totalHits, equalTo(3)); assertThat(topDocs.scoreDocs[0].doc, equalTo(2)); assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); @@ -323,7 +323,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN)))); // defaults to _last + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); // defaults to _last assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(7)); @@ -344,7 +344,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp // assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true))); // defaults to _last + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); // defaults to _last assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(6)); @@ -366,7 +366,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp searcher = new IndexSearcher(DirectoryReader.open(writer, true)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM)))); // defaults to _last + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM, null)))); // defaults to _last assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(7)); @@ -388,7 +388,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp searcher = new IndexSearcher(DirectoryReader.open(writer, true)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM), true))); // defaults to _last + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM, null), true))); // defaults to _last assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(6)); @@ -410,7 +410,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp searcher = new IndexSearcher(DirectoryReader.open(writer, true)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG)))); // defaults to _last + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG, null)))); // defaults to _last assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(7)); @@ -432,7 +432,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp searcher = new IndexSearcher(DirectoryReader.open(writer, true)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG), true))); // defaults to _last + new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG, null), true))); // defaults to _last assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(6)); @@ -453,7 +453,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp // assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN)))); + new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN, null)))); assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); @@ -466,7 +466,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp assertThat(topDocs.scoreDocs[7].doc, equalTo(6)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX), true))); + new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX, null), true))); assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); @@ -479,7 +479,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp assertThat(topDocs.scoreDocs[7].doc, equalTo(7)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("-9", MultiValueMode.MIN)))); + new Sort(new SortField("value", indexFieldData.comparatorSource("-9", MultiValueMode.MIN, null)))); assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(7)); @@ -492,7 +492,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp assertThat(topDocs.scoreDocs[7].doc, equalTo(6)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource("9", MultiValueMode.MAX), true))); + new Sort(new SortField("value", indexFieldData.comparatorSource("9", MultiValueMode.MAX, null), true))); assertThat(topDocs.totalHits, equalTo(8)); assertThat(topDocs.scoreDocs.length, equalTo(8)); assertThat(topDocs.scoreDocs[0].doc, equalTo(6)); diff --git a/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTests.java b/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTests.java index ac381aad419..d697ada619c 100644 --- a/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTests.java +++ b/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTests.java @@ -35,9 +35,9 @@ import org.elasticsearch.common.lucene.search.NotFilter; import org.elasticsearch.common.lucene.search.XFilteredQuery; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData; -import org.elasticsearch.index.search.nested.NestedFieldComparatorSource; import org.elasticsearch.search.MultiValueMode; import org.junit.Test; @@ -237,7 +237,7 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl final IndexFieldData indexFieldData = getForField("value"); final String missingValue = values[1]; IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); - XFieldComparatorSource comparator = indexFieldData.comparatorSource(missingValue, MultiValueMode.MIN); + XFieldComparatorSource comparator = indexFieldData.comparatorSource(missingValue, MultiValueMode.MIN, null); TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse))); assertEquals(numDocs, topDocs.totalHits); BytesRef previousValue = reverse ? UnicodeUtil.BIG_TERM : new BytesRef(); @@ -293,7 +293,7 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl } final IndexFieldData indexFieldData = getForField("value"); IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); - XFieldComparatorSource comparator = indexFieldData.comparatorSource(first ? "_first" : "_last", MultiValueMode.MIN); + XFieldComparatorSource comparator = indexFieldData.comparatorSource(first ? "_first" : "_last", MultiValueMode.MIN, null); TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse))); assertEquals(numDocs, topDocs.totalHits); BytesRef previousValue = first ? null : reverse ? UnicodeUtil.BIG_TERM : new BytesRef(); @@ -360,13 +360,13 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl } IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); IndexFieldData fieldData = getForField("text"); - final BytesRef missingValue; + final Object missingValue; switch (randomInt(4)) { case 0: - missingValue = new BytesRef(); + missingValue = "_first"; break; case 1: - missingValue = BytesRefFieldComparatorSource.MAX_TERM; + missingValue = "_last"; break; case 2: missingValue = new BytesRef(RandomPicks.randomFrom(getRandom(), values)); @@ -375,10 +375,10 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl missingValue = new BytesRef(TestUtil.randomSimpleString(getRandom())); break; } - BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode); Filter parentFilter = new TermFilter(new Term("type", "parent")); Filter childFilter = new NotFilter(parentFilter); - NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter); + Nested nested = new Nested(parentFilter, childFilter); + BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode, nested); ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("text", nestedComparatorSource)); TopFieldDocs topDocs = searcher.search(query, randomIntBetween(1, numParents), sort); @@ -389,26 +389,36 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl assertTrue("expected " + docID + " to be a parent", parents.get(docID)); BytesRef cmpValue = null; for (int child = parents.prevSetBit(docID - 1) + 1; child < docID; ++child) { - String[] vals = searcher.doc(child).getValues("text"); - if (vals.length == 0) { - vals = new String[] {missingValue.utf8ToString()}; + String[] sVals = searcher.doc(child).getValues("text"); + final BytesRef[] vals; + if (sVals.length == 0) { + vals = new BytesRef[0]; + } else { + vals = new BytesRef[sVals.length]; + for (int j = 0; j < vals.length; ++j) { + vals[j] = new BytesRef(sVals[j]); + } } - for (String value : vals) { - final BytesRef bytesValue = new BytesRef(value); + for (BytesRef value : vals) { if (cmpValue == null) { - cmpValue = bytesValue; - } else if (sortMode == MultiValueMode.MIN && bytesValue.compareTo(cmpValue) < 0) { - cmpValue = bytesValue; - } else if (sortMode == MultiValueMode.MAX && bytesValue.compareTo(cmpValue) > 0) { - cmpValue = bytesValue; + cmpValue = value; + } else if (sortMode == MultiValueMode.MIN && value.compareTo(cmpValue) < 0) { + cmpValue = value; + } else if (sortMode == MultiValueMode.MAX && value.compareTo(cmpValue) > 0) { + cmpValue = value; } } } if (cmpValue == null) { - cmpValue = missingValue; + if ("_first".equals(missingValue)) { + cmpValue = new BytesRef(); + } else if ("_last".equals(missingValue)) { + cmpValue = XFieldComparatorSource.MAX_TERM; + } else { + cmpValue = (BytesRef) missingValue; + } } if (previous != null) { - assertNotNull(cmpValue); assertTrue(previous.utf8ToString() + " / " + cmpValue.utf8ToString(), previous.compareTo(cmpValue) <= 0); } previous = cmpValue; diff --git a/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java b/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java index 2c13d685182..7bbb86da120 100644 --- a/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java +++ b/src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.elasticsearch.index.Index; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.mapper.FieldMapper.Names; import org.elasticsearch.search.MultiValueMode; @@ -60,8 +61,8 @@ public class NoOrdinalsStringFieldDataTests extends PagedBytesStringFieldDataTes } @Override - public XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) { - return new BytesRefFieldComparatorSource(this, missingValue, sortMode); + public XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) { + return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); } @Override diff --git a/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java b/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java index 75e398b37c3..852ae0eef7a 100644 --- a/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java +++ b/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.*; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; @@ -140,7 +141,7 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTests { public void testSorting() throws Exception { IndexFieldData indexFieldData = getForField(childType); IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); - IndexFieldData.XFieldComparatorSource comparator = indexFieldData.comparatorSource("_last", MultiValueMode.MIN); + IndexFieldData.XFieldComparatorSource comparator = indexFieldData.comparatorSource("_last", MultiValueMode.MIN, null); TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField(ParentFieldMapper.NAME, comparator, false))); assertThat(topDocs.totalHits, equalTo(8)); @@ -160,7 +161,7 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTests { assertThat(topDocs.scoreDocs[6].doc, equalTo(6)); assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("2")); assertThat(topDocs.scoreDocs[7].doc, equalTo(7)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]), equalTo(IndexFieldData.XFieldComparatorSource.MAX_TERM)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]), equalTo(XFieldComparatorSource.MAX_TERM)); topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField(ParentFieldMapper.NAME, comparator, true))); assertThat(topDocs.totalHits, equalTo(8)); diff --git a/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java b/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java index 96a1f6588f4..a14edde51ad 100644 --- a/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java +++ b/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java @@ -108,7 +108,7 @@ public class MultiOrdinalsTests extends ElasticsearchTestCase { }); Ordinals ords = creationMultiOrdinals(builder); RandomAccessOrds docs = ords.ordinals(); - final SortedDocValues singleOrds = MultiValueMode.MIN.select(docs, -1); + final SortedDocValues singleOrds = MultiValueMode.MIN.select(docs); int docId = ordsAndIds.get(0).id; List docOrds = new ArrayList<>(); for (OrdAndId ordAndId : ordsAndIds) { diff --git a/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTests.java b/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTests.java index 38159bbab58..08b75c6039a 100644 --- a/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTests.java +++ b/src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTests.java @@ -33,6 +33,8 @@ import org.elasticsearch.common.lucene.search.NotFilter; import org.elasticsearch.common.lucene.search.XFilteredQuery; import org.elasticsearch.index.fielddata.AbstractFieldDataTests; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.search.MultiValueMode; import org.junit.Test; @@ -206,10 +208,9 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData MultiValueMode sortMode = MultiValueMode.SUM; IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false)); - IndexFieldData.XFieldComparatorSource innerFieldComparator = createInnerFieldComparator("field2", sortMode, null); Filter parentFilter = new TermFilter(new Term("__type", "parent")); Filter childFilter = new NotFilter(parentFilter); - NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter); + XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, null, new Nested(parentFilter, childFilter)); ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("field2", nestedComparatorSource)); @@ -243,7 +244,7 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(9)); childFilter = new TermFilter(new Term("filter_1", "T")); - nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter); + nestedComparatorSource = createFieldComparator("field2", sortMode, null, new Nested(parentFilter, childFilter)); query = new ToParentBlockJoinQuery( new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), @@ -279,8 +280,7 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData assertThat(topDocs.scoreDocs[4].doc, equalTo(3)); assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(9)); - innerFieldComparator = createInnerFieldComparator("field2", sortMode, 127); - nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter); + nestedComparatorSource = createFieldComparator("field2", sortMode, 127, new Nested(parentFilter, childFilter)); sort = new Sort(new SortField("field2", nestedComparatorSource, true)); topDocs = searcher.search(new TermQuery(new Term("__type", "parent")), 5, sort); assertThat(topDocs.totalHits, equalTo(8)); @@ -296,8 +296,7 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData assertThat(topDocs.scoreDocs[4].doc, equalTo(7)); assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(8)); - innerFieldComparator = createInnerFieldComparator("field2", sortMode, -127); - nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter); + nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter)); sort = new Sort(new SortField("field2", nestedComparatorSource)); topDocs = searcher.search(new TermQuery(new Term("__type", "parent")), 5, sort); assertThat(topDocs.totalHits, equalTo(8)); @@ -315,33 +314,33 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData // Moved to method, because floating point based XFieldComparatorSource have different outcome for SortMode avg, // than integral number based implementations... - assertAvgScoreMode(parentFilter, searcher, innerFieldComparator); + assertAvgScoreMode(parentFilter, searcher); searcher.getIndexReader().close(); } - protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher, IndexFieldData.XFieldComparatorSource innerFieldComparator) throws IOException { + protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher) throws IOException { MultiValueMode sortMode = MultiValueMode.AVG; Filter childFilter = new NotFilter(parentFilter); - NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter); + XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter)); Query query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("field2", nestedComparatorSource)); TopDocs topDocs = searcher.search(query, 5, sort); assertThat(topDocs.totalHits, equalTo(7)); assertThat(topDocs.scoreDocs.length, equalTo(5)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(7)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(11)); assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(2)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(11)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(2)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(3)); + assertThat(topDocs.scoreDocs[1].doc, equalTo(3)); + assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(3)); + assertThat(topDocs.scoreDocs[2].doc, equalTo(7)); assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).intValue(), equalTo(3)); assertThat(topDocs.scoreDocs[3].doc, equalTo(15)); assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).intValue(), equalTo(3)); assertThat(topDocs.scoreDocs[4].doc, equalTo(19)); - assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(3)); + assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(4)); } protected abstract IndexableField createField(String name, int value, Field.Store store); - protected abstract IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue); + protected abstract IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested); } diff --git a/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java b/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java index 9c20eadfb62..cc2914219b8 100644 --- a/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java +++ b/src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java @@ -29,9 +29,11 @@ import org.elasticsearch.common.lucene.search.NotFilter; import org.elasticsearch.common.lucene.search.XFilteredQuery; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; -import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData; +import org.elasticsearch.search.MultiValueMode; import java.io.IOException; @@ -47,9 +49,9 @@ public class DoubleNestedSortingTests extends AbstractNumberNestedSortingTests { } @Override - protected IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue) { + protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) { DoubleArrayIndexFieldData fieldData = getForField(fieldName); - return new DoubleValuesComparatorSource(fieldData, missingValue, sortMode); + return new DoubleValuesComparatorSource(fieldData, missingValue, sortMode, nested); } @Override @@ -57,10 +59,10 @@ public class DoubleNestedSortingTests extends AbstractNumberNestedSortingTests { return new DoubleField(name, value, store); } - protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher, IndexFieldData.XFieldComparatorSource innerFieldComparator) throws IOException { + protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher) throws IOException { MultiValueMode sortMode = MultiValueMode.AVG; Filter childFilter = new NotFilter(parentFilter); - NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter); + XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter)); Query query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("field2", nestedComparatorSource)); TopDocs topDocs = searcher.search(query, 5, sort); diff --git a/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java b/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java index f00350f39c6..f652d11063d 100644 --- a/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java +++ b/src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java @@ -29,9 +29,11 @@ import org.elasticsearch.common.lucene.search.NotFilter; import org.elasticsearch.common.lucene.search.XFilteredQuery; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource; -import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.index.fielddata.plain.FloatArrayIndexFieldData; +import org.elasticsearch.search.MultiValueMode; import java.io.IOException; @@ -39,7 +41,7 @@ import static org.hamcrest.Matchers.equalTo; /** */ -public class FloatNestedSortingTests extends AbstractNumberNestedSortingTests { +public class FloatNestedSortingTests extends DoubleNestedSortingTests { @Override protected FieldDataType getFieldDataType() { @@ -47,9 +49,9 @@ public class FloatNestedSortingTests extends AbstractNumberNestedSortingTests { } @Override - protected IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue) { + protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) { FloatArrayIndexFieldData fieldData = getForField(fieldName); - return new FloatValuesComparatorSource(fieldData, missingValue, sortMode); + return new FloatValuesComparatorSource(fieldData, missingValue, sortMode, nested); } @Override @@ -60,7 +62,7 @@ public class FloatNestedSortingTests extends AbstractNumberNestedSortingTests { protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher, IndexFieldData.XFieldComparatorSource innerFieldComparator) throws IOException { MultiValueMode sortMode = MultiValueMode.AVG; Filter childFilter = new NotFilter(parentFilter); - NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter); + XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter)); Query query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("field2", nestedComparatorSource)); TopDocs topDocs = searcher.search(query, 5, sort); diff --git a/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java b/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java index accdfb7734b..927aa672019 100644 --- a/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java +++ b/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java @@ -23,9 +23,10 @@ import org.apache.lucene.document.LongField; import org.apache.lucene.index.IndexableField; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; -import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.index.fielddata.plain.PackedArrayIndexFieldData; +import org.elasticsearch.search.MultiValueMode; /** */ @@ -37,9 +38,9 @@ public class LongNestedSortingTests extends AbstractNumberNestedSortingTests { } @Override - protected IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue) { + protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) { PackedArrayIndexFieldData fieldData = getForField(fieldName); - return new LongValuesComparatorSource(fieldData, missingValue, sortMode); + return new LongValuesComparatorSource(fieldData, missingValue, sortMode, nested); } @Override diff --git a/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java index 05a8d328d8f..7db34b057ee 100644 --- a/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java +++ b/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.lucene.search.XFilteredQuery; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.index.fielddata.*; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.search.MultiValueMode; @@ -106,8 +107,7 @@ public class NestedSortingTests extends AbstractFieldDataTests { private TopDocs getTopDocs(IndexSearcher searcher, IndexFieldData indexFieldData, String missingValue, MultiValueMode sortMode, int n, boolean reverse) throws IOException { Filter parentFilter = new TermFilter(new Term("__type", "parent")); Filter childFilter = new TermFilter(new Term("__type", "child")); - XFieldComparatorSource innerSource = indexFieldData.comparatorSource(missingValue, sortMode); - NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter); + XFieldComparatorSource nestedComparatorSource = indexFieldData.comparatorSource(missingValue, sortMode, new Nested(parentFilter, childFilter)); Query query = new ConstantScoreQuery(parentFilter); Sort sort = new Sort(new SortField("f", nestedComparatorSource, reverse)); return searcher.search(query, n, sort); @@ -271,10 +271,9 @@ public class NestedSortingTests extends AbstractFieldDataTests { MultiValueMode sortMode = MultiValueMode.MIN; IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false)); PagedBytesIndexFieldData indexFieldData = getForField("field2"); - BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode); Filter parentFilter = new TermFilter(new Term("__type", "parent")); Filter childFilter = new NotFilter(parentFilter); - NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter); + BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, new Nested(parentFilter, childFilter)); ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None); Sort sort = new Sort(new SortField("field2", nestedComparatorSource)); @@ -293,7 +292,7 @@ public class NestedSortingTests extends AbstractFieldDataTests { assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("i")); sortMode = MultiValueMode.MAX; - nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter); + nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, new Nested(parentFilter, childFilter)); sort = new Sort(new SortField("field2", nestedComparatorSource, true)); topDocs = searcher.search(query, 5, sort); assertThat(topDocs.totalHits, equalTo(7)); @@ -311,7 +310,7 @@ public class NestedSortingTests extends AbstractFieldDataTests { childFilter = new AndFilter(Arrays.asList(new NotFilter(parentFilter), new TermFilter(new Term("filter_1", "T")))); - nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter); + nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, new Nested(parentFilter, childFilter)); query = new ToParentBlockJoinQuery( new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), diff --git a/src/test/java/org/elasticsearch/nested/SimpleNestedTests.java b/src/test/java/org/elasticsearch/nested/SimpleNestedTests.java index e7270ba29f3..fd5816b8e42 100644 --- a/src/test/java/org/elasticsearch/nested/SimpleNestedTests.java +++ b/src/test/java/org/elasticsearch/nested/SimpleNestedTests.java @@ -1263,11 +1263,11 @@ public class SimpleNestedTests extends ElasticsearchIntegrationTest { assertHitCount(searchResponse, 3); assertThat(searchResponse.getHits().getHits().length, equalTo(3)); assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("3")); - assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("0")); + assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("1")); assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2")); - assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("1")); + assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2")); assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("1")); - assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("2")); + assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("3")); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -1282,11 +1282,11 @@ public class SimpleNestedTests extends ElasticsearchIntegrationTest { assertHitCount(searchResponse, 3); assertThat(searchResponse.getHits().getHits().length, equalTo(3)); assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1")); - assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("2")); + assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("3")); assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2")); - assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("1")); + assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2")); assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3")); - assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("0")); + assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("1")); // Sort mode: avg with filter searchResponse = client().prepareSearch() diff --git a/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/src/test/java/org/elasticsearch/search/MultiValueModeTests.java new file mode 100644 index 00000000000..7fe4f31b530 --- /dev/null +++ b/src/test/java/org/elasticsearch/search/MultiValueModeTests.java @@ -0,0 +1,545 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; +import org.apache.lucene.index.*; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.NumericDoubleValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.test.ElasticsearchTestCase; + +import java.util.Arrays; + +public class MultiValueModeTests extends ElasticsearchTestCase { + + private static FixedBitSet randomRootDocs(int maxDoc) { + FixedBitSet set = new FixedBitSet(maxDoc); + for (int i = 0; i < maxDoc; ++i) { + if (randomBoolean()) { + set.set(i); + } + } + // the last doc must be a root doc + set.set(maxDoc - 1); + return set; + } + + private static FixedBitSet randomInnerDocs(FixedBitSet rootDocs) { + FixedBitSet innerDocs = new FixedBitSet(rootDocs.length()); + for (int i = 0; i < innerDocs.length(); ++i) { + if (!rootDocs.get(i) && randomBoolean()) { + innerDocs.set(i); + } + } + return innerDocs; + } + + public void testSingleValuedLongs() { + final int numDocs = scaledRandomIntBetween(1, 100); + final long[] array = new long[numDocs]; + final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); + for (int i = 0; i < array.length; ++i) { + if (randomBoolean()) { + array[i] = randomLong(); + if (docsWithValue != null) { + docsWithValue.set(i); + } + } else if (docsWithValue != null && randomBoolean()) { + docsWithValue.set(i); + } + } + final NumericDocValues singleValues = new NumericDocValues() { + @Override + public long get(int docID) { + return array[docID]; + } + }; + final SortedNumericDocValues multiValues = DocValues.singleton(singleValues, docsWithValue); + verify(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verify(multiValues, numDocs, rootDocs, innerDocs); + } + + public void testMultiValuedLongs() { + final int numDocs = scaledRandomIntBetween(1, 100); + final long[][] array = new long[numDocs][]; + for (int i = 0; i < numDocs; ++i) { + final long[] values = new long[randomInt(4)]; + for (int j = 0; j < values.length; ++j) { + values[j] = randomLong(); + } + Arrays.sort(values); + array[i] = values; + } + final SortedNumericDocValues multiValues = new SortedNumericDocValues() { + int doc; + + @Override + public long valueAt(int index) { + return array[doc][index]; + } + + @Override + public void setDocument(int doc) { + this.doc = doc; + } + + @Override + public int count() { + return array[doc].length; + } + }; + verify(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verify(multiValues, numDocs, rootDocs, innerDocs); + } + + private void verify(SortedNumericDocValues values, int maxDoc) { + for (long missingValue : new long[] { 0, randomLong() }) { + for (MultiValueMode mode : MultiValueMode.values()) { + final NumericDocValues selected = mode.select(values, missingValue); + for (int i = 0; i < maxDoc; ++i) { + final long actual = selected.get(i); + long expected; + values.setDocument(i); + int numValues = values.count(); + if (numValues == 0) { + expected = missingValue; + } else { + expected = mode.startLong(); + for (int j = 0; j < numValues; ++j) { + expected = mode.apply(expected, values.valueAt(j)); + } + expected = mode.reduce(expected, numValues); + } + + assertEquals(mode.toString() + " docId=" + i, expected, actual); + } + } + } + } + + private void verify(SortedNumericDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) { + for (long missingValue : new long[] { 0, randomLong() }) { + for (MultiValueMode mode : MultiValueMode.values()) { + final NumericDocValues selected = mode.select(values, missingValue, rootDocs, innerDocs, maxDoc); + int prevRoot = -1; + for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { + final long actual = selected.get(root); + long expected = mode.startLong(); + int numValues = 0; + for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { + values.setDocument(child); + for (int j = 0; j < values.count(); ++j) { + expected = mode.apply(expected, values.valueAt(j)); + ++numValues; + } + } + if (numValues == 0) { + expected = missingValue; + } else { + expected = mode.reduce(expected, numValues); + } + + assertEquals(mode.toString() + " docId=" + root, expected, actual); + + prevRoot = root; + } + } + } + } + + public void testSingleValuedDoubles() { + final int numDocs = scaledRandomIntBetween(1, 100); + final double[] array = new double[numDocs]; + final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); + for (int i = 0; i < array.length; ++i) { + if (randomBoolean()) { + array[i] = randomDouble(); + if (docsWithValue != null) { + docsWithValue.set(i); + } + } else if (docsWithValue != null && randomBoolean()) { + docsWithValue.set(i); + } + } + final NumericDoubleValues singleValues = new NumericDoubleValues() { + @Override + public double get(int docID) { + return array[docID]; + } + }; + final SortedNumericDoubleValues multiValues = FieldData.singleton(singleValues, docsWithValue); + verify(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verify(multiValues, numDocs, rootDocs, innerDocs); + } + + public void testMultiValuedDoubles() { + final int numDocs = scaledRandomIntBetween(1, 100); + final double[][] array = new double[numDocs][]; + for (int i = 0; i < numDocs; ++i) { + final double[] values = new double[randomInt(4)]; + for (int j = 0; j < values.length; ++j) { + values[j] = randomDouble(); + } + Arrays.sort(values); + array[i] = values; + } + final SortedNumericDoubleValues multiValues = new SortedNumericDoubleValues() { + int doc; + + @Override + public double valueAt(int index) { + return array[doc][index]; + } + + @Override + public void setDocument(int doc) { + this.doc = doc; + } + + @Override + public int count() { + return array[doc].length; + } + }; + verify(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verify(multiValues, numDocs, rootDocs, innerDocs); + } + + private void verify(SortedNumericDoubleValues values, int maxDoc) { + for (long missingValue : new long[] { 0, randomLong() }) { + for (MultiValueMode mode : MultiValueMode.values()) { + final NumericDoubleValues selected = mode.select(values, missingValue); + for (int i = 0; i < maxDoc; ++i) { + final double actual = selected.get(i); + double expected; + values.setDocument(i); + int numValues = values.count(); + if (numValues == 0) { + expected = missingValue; + } else { + expected = mode.startLong(); + for (int j = 0; j < numValues; ++j) { + expected = mode.apply(expected, values.valueAt(j)); + } + expected = mode.reduce(expected, numValues); + } + + assertEquals(mode.toString() + " docId=" + i, expected, actual, 0.1); + } + } + } + } + + private void verify(SortedNumericDoubleValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) { + for (long missingValue : new long[] { 0, randomLong() }) { + for (MultiValueMode mode : MultiValueMode.values()) { + final NumericDoubleValues selected = mode.select(values, missingValue, rootDocs, innerDocs, maxDoc); + int prevRoot = -1; + for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { + final double actual = selected.get(root); + double expected = mode.startLong(); + int numValues = 0; + for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { + values.setDocument(child); + for (int j = 0; j < values.count(); ++j) { + expected = mode.apply(expected, values.valueAt(j)); + ++numValues; + } + } + if (numValues == 0) { + expected = missingValue; + } else { + expected = mode.reduce(expected, numValues); + } + + assertEquals(mode.toString() + " docId=" + root, expected, actual, 0.1); + + prevRoot = root; + } + } + } + } + + public void testSingleValuedStrings() { + final int numDocs = scaledRandomIntBetween(1, 100); + final BytesRef[] array = new BytesRef[numDocs]; + final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); + for (int i = 0; i < array.length; ++i) { + if (randomBoolean()) { + array[i] = new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8)); + if (docsWithValue != null) { + docsWithValue.set(i); + } + } else { + array[i] = new BytesRef(); + if (docsWithValue != null && randomBoolean()) { + docsWithValue.set(i); + } + } + } + final BinaryDocValues singleValues = new BinaryDocValues() { + @Override + public BytesRef get(int docID) { + return BytesRef.deepCopyOf(array[docID]); + } + }; + final SortedBinaryDocValues multiValues = FieldData.singleton(singleValues, docsWithValue); + verify(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verify(multiValues, numDocs, rootDocs, innerDocs); + } + + public void testMultiValuedStrings() { + final int numDocs = scaledRandomIntBetween(1, 100); + final BytesRef[][] array = new BytesRef[numDocs][]; + for (int i = 0; i < numDocs; ++i) { + final BytesRef[] values = new BytesRef[randomInt(4)]; + for (int j = 0; j < values.length; ++j) { + values[j] = new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8)); + } + Arrays.sort(values); + array[i] = values; + } + final SortedBinaryDocValues multiValues = new SortedBinaryDocValues() { + int doc; + + @Override + public BytesRef valueAt(int index) { + return BytesRef.deepCopyOf(array[doc][index]); + } + + @Override + public void setDocument(int doc) { + this.doc = doc; + } + + @Override + public int count() { + return array[doc].length; + } + }; + verify(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verify(multiValues, numDocs, rootDocs, innerDocs); + } + + private void verify(SortedBinaryDocValues values, int maxDoc) { + for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8)) }) { + for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + final BinaryDocValues selected = mode.select(values, missingValue); + for (int i = 0; i < maxDoc; ++i) { + final BytesRef actual = selected.get(i); + BytesRef expected = null; + values.setDocument(i); + int numValues = values.count(); + if (numValues == 0) { + expected = missingValue; + } else { + for (int j = 0; j < numValues; ++j) { + if (expected == null) { + expected = BytesRef.deepCopyOf(values.valueAt(j)); + } else { + expected = mode.apply(expected, BytesRef.deepCopyOf(values.valueAt(j))); + } + } + if (expected == null) { + expected = missingValue; + } + } + + assertEquals(mode.toString() + " docId=" + i, expected, actual); + } + } + } + } + + private void verify(SortedBinaryDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) { + for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8)) }) { + for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + final BinaryDocValues selected = mode.select(values, missingValue, rootDocs, innerDocs, maxDoc); + int prevRoot = -1; + for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { + final BytesRef actual = selected.get(root); + BytesRef expected = null; + for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { + values.setDocument(child); + for (int j = 0; j < values.count(); ++j) { + if (expected == null) { + expected = BytesRef.deepCopyOf(values.valueAt(j)); + } else { + expected = mode.apply(expected, values.valueAt(j)); + } + expected = mode.apply(expected, BytesRef.deepCopyOf(values.valueAt(j))); + } + } + if (expected == null) { + expected = missingValue; + } + + assertEquals(mode.toString() + " docId=" + root, expected, actual); + + prevRoot = root; + } + } + } + } + + + public void testSingleValuedOrds() { + final int numDocs = scaledRandomIntBetween(1, 100); + final int[] array = new int[numDocs]; + for (int i = 0; i < array.length; ++i) { + if (randomBoolean()) { + array[i] = randomInt(1000); + } else { + array[i] = -1; + } + } + final SortedDocValues singleValues = new SortedDocValues() { + @Override + public int getOrd(int docID) { + return array[docID]; + } + + @Override + public BytesRef lookupOrd(int ord) { + throw new UnsupportedOperationException(); + } + + @Override + public int getValueCount() { + return 1 << 20; + } + }; + final RandomAccessOrds multiValues = (RandomAccessOrds) DocValues.singleton(singleValues); + verify(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verify(multiValues, numDocs, rootDocs, innerDocs); + } + + public void testMultiValuedOrds() { + final int numDocs = scaledRandomIntBetween(1, 100); + final long[][] array = new long[numDocs][]; + for (int i = 0; i < numDocs; ++i) { + final long[] values = new long[randomInt(4)]; + for (int j = 0; j < values.length; ++j) { + values[j] = j == 0 ? randomInt(1000) : values[j - 1] + 1 + randomInt(1000); + } + array[i] = values; + } + final RandomAccessOrds multiValues = new RandomAccessOrds() { + int doc; + + @Override + public long ordAt(int index) { + return array[doc][index]; + } + + @Override + public int cardinality() { + return array[doc].length; + } + + @Override + public long nextOrd() { + throw new UnsupportedOperationException(); + } + + @Override + public void setDocument(int docID) { + this.doc = docID; + } + + @Override + public BytesRef lookupOrd(long ord) { + throw new UnsupportedOperationException(); + } + + @Override + public long getValueCount() { + return 1 << 20; + } + }; + verify(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verify(multiValues, numDocs, rootDocs, innerDocs); + } + + private void verify(RandomAccessOrds values, int maxDoc) { + for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + final SortedDocValues selected = mode.select(values); + for (int i = 0; i < maxDoc; ++i) { + final long actual = selected.getOrd(i); + int expected = -1; + values.setDocument(i); + for (int j = 0; j < values.cardinality(); ++j) { + if (expected == -1) { + expected = (int) values.ordAt(j); + } else { + expected = mode.applyOrd(expected, (int) values.ordAt(j)); + } + } + + assertEquals(mode.toString() + " docId=" + i, expected, actual); + } + } + } + + private void verify(RandomAccessOrds values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) { + for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + final SortedDocValues selected = mode.select(values, rootDocs, innerDocs); + int prevRoot = -1; + for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { + final int actual = selected.getOrd(root); + int expected = -1; + for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { + values.setDocument(child); + for (int j = 0; j < values.cardinality(); ++j) { + if (expected == -1) { + expected = (int) values.ordAt(j); + } else { + expected = mode.applyOrd(expected, (int) values.ordAt(j)); + } + } + } + + assertEquals(mode.toString() + " docId=" + root, expected, actual); + + prevRoot = root; + } + } + } +} diff --git a/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java b/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java index 97e6c9b8bd8..e1df0fa2b64 100644 --- a/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java +++ b/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java @@ -23,6 +23,7 @@ import com.carrotsearch.randomizedtesting.annotations.*; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.AbstractRandomizedTest; import org.apache.lucene.util.LuceneTestCase; @@ -39,10 +40,7 @@ import org.elasticsearch.test.cache.recycler.MockBigArrays; import org.elasticsearch.test.cache.recycler.MockPageCacheRecycler; import org.elasticsearch.test.junit.listeners.LoggingListener; import org.elasticsearch.test.store.MockDirectoryHelper; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Ignore; +import org.junit.*; import java.io.Closeable; import java.io.File; @@ -99,6 +97,21 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest { } + @Before + public void cleanFieldCache() { + FieldCache.DEFAULT.purgeAllCaches(); + } + + @After + public void ensureNoFieldCacheUse() { + // We use the lucene comparators, and by default they work on field cache. + // However, given the way that we use them, field cache should NEVER get loaded. + if (getClass().getAnnotation(UsesLuceneFieldCacheOnPurpose.class) == null) { + FieldCache.CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries(); + assertEquals("fieldcache must never be used, got=" + Arrays.toString(entries), 0, entries.length); + } + } + /** * Runs the code block for 10 seconds waiting for no assertion to trip. */ @@ -395,6 +408,15 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest { int version(); } + /** + * Most tests don't use {@link FieldCache} but some of them might do. + */ + @Retention(RetentionPolicy.RUNTIME) + @Target({ElementType.TYPE}) + @Ignore + public @interface UsesLuceneFieldCacheOnPurpose { + } + /** * Returns a global compatibility version that is set via the * {@value #TESTS_COMPATIBILITY} or {@value #TESTS_BACKWARDS_COMPATIBILITY_VERSION} system property.