honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fielddata: goodbye comparators. 

This commit removes custom comparators in favor of the ones that are in Lucene.

The major change is for nested documents: instead of having a comparator wrapper
that deals with nested documents, this is done at the fielddata level by having
a selector that returns the value to use for comparison.

Sorting with custom missing string values might be slower since it is using
TermValComparator since Lucene's TermOrdValComparator only supports sorting
missing values first or last. But other than this particular case, this change
will allow us to benefit from improvements on comparators from the Lucene side.

Close #5980
This commit is contained in:
Adrien Grand 2014-07-18 17:02:07 +02:00
parent a1a03a184c
commit 629f91ae57
52 changed files with 1509 additions and 2011 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/org/elasticsearch/index/fielddata/FieldData.java
View File

@ -40,6 +40,13 @@ public enum FieldData {
assert Lucene.VERSION == Version.LUCENE_4_9 : "Remove emptySortedNumeric in 4.10 and use the method with the same name from Lucene's DocValues class. See LUCENE-5834.";
}
/**
* Return a {@link SortedBinaryDocValues} that doesn't contain any value.
*/
public static SortedBinaryDocValues emptySortedBinary(int maxDoc) {
return singleton(DocValues.emptyBinary(), new Bits.MatchNoBits(maxDoc));
}
/**
* Return a {@link SortedNumericDocValues} that doesn't contain any value.
*/

55
src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
View File

@ -21,20 +21,23 @@ package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.FieldComparatorSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.*;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.UnicodeUtil;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexComponent;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.fielddata.breaker.CircuitBreakerService;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
* Thread-safe utility class that allows to get per-segment values via the
* {@link #load(AtomicReaderContext)} method.
@ -93,7 +96,7 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
/**
* Comparator used for sorting.
*/
XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode);
XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested);
/**
* Clears any resources associated with this field data.
@ -116,6 +119,52 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
UnicodeUtil.UTF16toUTF8(chars, 0, chars.length, MAX_TERM);
}
/**
* Simple wrapper class around a filter that matches parent documents
* and a filter that matches child documents. For every root document R,
* R will be in the parent filter and its children documents will be the
* documents that are contained in the inner set between the previous
* parent + 1, or 0 if there is no previous parent, and R (excluded).
*/
public static class Nested {
private final Filter rootFilter, innerFilter;
public Nested(Filter rootFilter, Filter innerFilter) {
this.rootFilter = rootFilter;
this.innerFilter = innerFilter;
}
// TODO: nested docs should not be random filters but specialized
// ones that guarantee that you always get a FixedBitSet
@Deprecated
private static FixedBitSet toFixedBitSet(DocIdSet set, int maxDoc) throws IOException {
if (set == null || set instanceof FixedBitSet) {
return (FixedBitSet) set;
} else {
final FixedBitSet fixedBitSet = new FixedBitSet(maxDoc);
final DocIdSetIterator it = set.iterator();
if (it != null) {
fixedBitSet.or(it);
}
return fixedBitSet;
}
}
/**
* Get a {@link FixedBitSet} that matches the root documents.
*/
public FixedBitSet rootDocs(AtomicReaderContext ctx) throws IOException {
return toFixedBitSet(rootFilter.getDocIdSet(ctx, null), ctx.reader().maxDoc());
}
/**
* Get a {@link FixedBitSet} that matches the inner documents.
*/
public FixedBitSet innerDocs(AtomicReaderContext ctx) throws IOException {
return toFixedBitSet(innerFilter.getDocIdSet(ctx, null), ctx.reader().maxDoc());
}
}
/** Whether missing values should be sorted first. */
protected final boolean sortMissingFirst(Object missingValue) {
return "_first".equals(missingValue);

248
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
View File

@ -19,37 +19,41 @@
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.Version;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
* Comparator source for string/binary values.
*/
public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparatorSource {
/** UTF-8 term containing a single code point: {@link Character#MAX_CODE_POINT} which will compare greater than all other index terms
* since {@link Character#MAX_CODE_POINT} is a noncharacter and thus shouldn't appear in an index term. */
public static final BytesRef MAX_TERM;
static {
MAX_TERM = new BytesRef();
final char[] chars = Character.toChars(Character.MAX_CODE_POINT);
UnicodeUtil.UTF16toUTF8(chars, 0, chars.length, MAX_TERM);
}
private final IndexFieldData<?> indexFieldData;
private final MultiValueMode sortMode;
private final Object missingValue;
private final Nested nested;
public BytesRefFieldComparatorSource(IndexFieldData<?> indexFieldData, Object missingValue, MultiValueMode sortMode) {
public BytesRefFieldComparatorSource(IndexFieldData<?> indexFieldData, Object missingValue, MultiValueMode sortMode, Nested nested) {
this.indexFieldData = indexFieldData;
this.sortMode = sortMode;
this.missingValue = missingValue;
this.nested = nested;
}
@Override
@ -57,15 +61,229 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat
return SortField.Type.STRING;
}
protected SortedBinaryDocValues getValues(AtomicReaderContext context) {
return indexFieldData.load(context).getBytesValues();
}
protected void setScorer(Scorer scorer) {}
@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
assert fieldname.equals(indexFieldData.getFieldNames().indexName());
final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed);
assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName());
final boolean sortMissingLast = sortMissingLast(missingValue) ^ reversed;
final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed);
if (indexFieldData instanceof IndexOrdinalsFieldData) {
return new BytesRefOrdValComparator((IndexOrdinalsFieldData) indexFieldData, numHits, sortMode, missingBytes);
// The ordinal-based comparator only supports sorting missing values first or last so when
// a missing value is provided we fall back to the (slow) value-based comparator
// TODO: handle arbitrary missing values via a selector
if (sortMissingFirst(missingValue) || sortMissingLast(missingValue)) {
return new FieldComparator.TermOrdValComparator(numHits, null, sortMissingLast) {
@Override
protected SortedDocValues getSortedDocValues(AtomicReaderContext context, String field) throws IOException {
final RandomAccessOrds values = ((IndexOrdinalsFieldData) indexFieldData).load(context).getOrdinalsValues();
final SortedDocValues selectedValues;
if (nested == null) {
selectedValues = sortMode.select(values);
} else {
final FixedBitSet rootDocs = nested.rootDocs(context);
final FixedBitSet innerDocs = nested.innerDocs(context);
selectedValues = sortMode.select(values, rootDocs, innerDocs);
}
return selectedValues;
}
public BytesRef value(int slot) {
// TODO: When serializing the response to the coordinating node, we lose the information about
// whether the comparator sorts missing docs first or last. We should fix it and let
// TopDocs.merge deal with it (it knows how to)
BytesRef value = super.value(slot);
if (value == null) {
value = missingBytes;
}
return value;
}
};
}
}
return new BytesRefValComparator(indexFieldData, numHits, sortMode, missingBytes);
final BytesRef nullPlaceHolder = new BytesRef();
final BytesRef nonNullMissingBytes = missingBytes == null ? nullPlaceHolder : missingBytes;
return new TermValComparator(numHits, null, sortMissingLast) {
@Override
protected BinaryDocValues getBinaryDocValues(AtomicReaderContext context, String field) throws IOException {
final SortedBinaryDocValues values = getValues(context);
final BinaryDocValues selectedValues;
if (nested == null) {
selectedValues = sortMode.select(values, nonNullMissingBytes);
} else {
final FixedBitSet rootDocs = nested.rootDocs(context);
final FixedBitSet innerDocs = nested.innerDocs(context);
selectedValues = sortMode.select(values, nonNullMissingBytes, rootDocs, innerDocs, context.reader().maxDoc());
}
return selectedValues;
}
@Override
protected Bits getDocsWithField(AtomicReaderContext context, String field) throws IOException {
return new Bits.MatchAllBits(context.reader().maxDoc());
}
@Override
protected boolean isNull(int doc, BytesRef term) {
return term == nullPlaceHolder;
}
@Override
public void setScorer(Scorer scorer) {
BytesRefFieldComparatorSource.this.setScorer(scorer);
}
@Override
public BytesRef value(int slot) {
BytesRef value = super.value(slot);
if (value == null) {
value = missingBytes;
}
return value;
}
};
}
static {
assert Lucene.VERSION == Version.LUCENE_4_9 : "The comparator below is a raw copy of Lucene's, remove it when upgrading to 4.10";
}
/** Sorts by field's natural Term sort order. All
* comparisons are done using BytesRef.compareTo, which is
* slow for medium to large result sets but possibly
* very fast for very small results sets. */
public static class TermValComparator extends FieldComparator<BytesRef> {
private final BytesRef[] values;
private final BytesRef[] tempBRs;
private BinaryDocValues docTerms;
private Bits docsWithField;
private final String field;
private BytesRef bottom;
private BytesRef topValue;
private final int missingSortCmp;
/** Sole constructor. */
public TermValComparator(int numHits, String field, boolean sortMissingLast) {
values = new BytesRef[numHits];
tempBRs = new BytesRef[numHits];
this.field = field;
missingSortCmp = sortMissingLast ? 1 : -1;
}
@Override
public int compare(int slot1, int slot2) {
final BytesRef val1 = values[slot1];
final BytesRef val2 = values[slot2];
return compareValues(val1, val2);
}
@Override
public int compareBottom(int doc) {
final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
return compareValues(bottom, comparableBytes);
}
@Override
public void copy(int slot, int doc) {
final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
if (comparableBytes == null) {
values[slot] = null;
} else {
if (tempBRs[slot] == null) {
tempBRs[slot] = new BytesRef();
}
values[slot] = tempBRs[slot];
values[slot].copyBytes(comparableBytes);
}
}
/** Retrieves the BinaryDocValues for the field in this segment */
protected BinaryDocValues getBinaryDocValues(AtomicReaderContext context, String field) throws IOException {
return FieldCache.DEFAULT.getTerms(context.reader(), field, true);
}
/** Retrieves the set of documents that have a value in this segment */
protected Bits getDocsWithField(AtomicReaderContext context, String field) throws IOException {
return FieldCache.DEFAULT.getDocsWithField(context.reader(), field);
}
/** Check whether the given value represents <tt>null</tt>. This can be
* useful if the {@link BinaryDocValues} returned by {@link #getBinaryDocValues}
* use a special value as a sentinel. The default implementation checks
* {@link #getDocsWithField}.
* <p>NOTE: The null value can only be an EMPTY {@link BytesRef}. */
protected boolean isNull(int doc, BytesRef term) {
return docsWithField != null && docsWithField.get(doc) == false;
}
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
docTerms = getBinaryDocValues(context, field);
docsWithField = getDocsWithField(context, field);
if (docsWithField instanceof Bits.MatchAllBits) {
docsWithField = null;
}
return this;
}
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
}
@Override
public void setTopValue(BytesRef value) {
// null is fine: it means the last doc of the prior
// search was missing this value
topValue = value;
}
@Override
public BytesRef value(int slot) {
return values[slot];
}
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
// missing always sorts first:
if (val1 == null) {
if (val2 == null) {
return 0;
}
return missingSortCmp;
} else if (val2 == null) {
return -missingSortCmp;
}
return val1.compareTo(val2);
}
@Override
public int compareTop(int doc) {
final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
return compareValues(topValue, comparableBytes);
}
/**
* Given a document and a term, return the term itself if it exists or
* <tt>null</tt> otherwise.
*/
private BytesRef getComparableBytes(int doc, BytesRef term) {
if (term.length == 0 && isNull(doc, term)) {
return null;
}
return term;
}
}
}

339
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java
View File

@ -1,339 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
* Sorts by field's natural Term sort order, using
* ordinals. This is functionally equivalent to {@link
* org.apache.lucene.search.FieldComparator.TermValComparator}, but it first resolves the string
* to their relative ordinal positions (using the index
* returned by {@link org.apache.lucene.search.FieldCache#getTermsIndex}), and
* does most comparisons using the ordinals. For medium
* to large results, this comparator will be much faster
* than {@link org.apache.lucene.search.FieldComparator.TermValComparator}. For very small
* result sets it may be slower.
*
* Internally this comparator multiplies ordinals by 4 so that virtual ordinals can be inserted in-between the original field data ordinals.
* Thanks to this, an ordinal for the missing value and the bottom value can be computed and all ordinals are directly comparable. For example,
* if the field data ordinals are (a,1), (b,2) and (c,3), they will be internally stored as (a,4), (b,8), (c,12). Then the ordinal for the
* missing value will be computed by binary searching. For example, if the missing value is 'ab', it will be assigned 6 as an ordinal (between
* 'a' and 'b'. And if the bottom value is 'ac', it will be assigned 7 as an ordinal (between 'ab' and 'b').
*/
public final class BytesRefOrdValComparator extends NestedWrappableComparator<BytesRef> {
final IndexOrdinalsFieldData indexFieldData;
final BytesRef missingValue;
/* Ords for each slot, times 4.
@lucene.internal */
final long[] ords;
final MultiValueMode sortMode;
/* Values for each slot.
@lucene.internal */
final BytesRef[] values;
/* Which reader last copied a value into the slot. When
we compare two slots, we just compare-by-ord if the
readerGen is the same; else we must compare the
values (slower).
@lucene.internal */
final int[] readerGen;
/* Gen of current reader we are on.
@lucene.internal */
int currentReaderGen = -1;
/* Current reader's doc ord/values.
@lucene.internal */
SortedDocValues termsIndex;
long missingOrd;
/* Bottom slot, or -1 if queue isn't full yet
@lucene.internal */
int bottomSlot = -1;
/* Bottom ord (same as ords[bottomSlot] once bottomSlot
is set). Cached for faster compares.
@lucene.internal */
long bottomOrd;
BytesRef top;
long topOrd;
public BytesRefOrdValComparator(IndexOrdinalsFieldData indexFieldData, int numHits, MultiValueMode sortMode, BytesRef missingValue) {
this.indexFieldData = indexFieldData;
this.sortMode = sortMode;
this.missingValue = missingValue;
ords = new long[numHits];
values = new BytesRef[numHits];
readerGen = new int[numHits];
}
@Override
public int compare(int slot1, int slot2) {
if (readerGen[slot1] == readerGen[slot2]) {
final int res = Long.compare(ords[slot1], ords[slot2]);
assert Integer.signum(res) == Integer.signum(compareValues(values[slot1], values[slot2])) : values[slot1] + " " + values[slot2] + " " + ords[slot1] + " " + ords[slot2];
return res;
}
final BytesRef val1 = values[slot1];
final BytesRef val2 = values[slot2];
return compareValues(val1, val2);
}
@Override
public int compareBottom(int doc) {
throw new UnsupportedOperationException();
}
@Override
public int compareTop(int doc) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public int compareBottomMissing() {
throw new UnsupportedOperationException();
}
@Override
public void copy(int slot, int doc) {
throw new UnsupportedOperationException();
}
@Override
public void missing(int slot) {
throw new UnsupportedOperationException();
}
@Override
public int compareTopMissing() {
throw new UnsupportedOperationException();
}
class PerSegmentComparator extends NestedWrappableComparator<BytesRef> {
final SortedDocValues termsIndex;
public PerSegmentComparator(SortedDocValues termsIndex) {
this.termsIndex = termsIndex;
}
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
return BytesRefOrdValComparator.this.setNextReader(context);
}
@Override
public int compare(int slot1, int slot2) {
return BytesRefOrdValComparator.this.compare(slot1, slot2);
}
@Override
public void setBottom(final int bottom) {
BytesRefOrdValComparator.this.setBottom(bottom);
}
@Override
public void setTopValue(BytesRef value) {
BytesRefOrdValComparator.this.setTopValue(value);
}
@Override
public BytesRef value(int slot) {
return BytesRefOrdValComparator.this.value(slot);
}
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
if (val1 == null) {
if (val2 == null) {
return 0;
}
return -1;
} else if (val2 == null) {
return 1;
}
return val1.compareTo(val2);
}
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
final long docOrd = termsIndex.getOrd(doc);
final long comparableOrd = docOrd < 0 ? missingOrd : docOrd << 2;
return Long.compare(bottomOrd, comparableOrd);
}
@Override
public int compareTop(int doc) throws IOException {
final long ord = termsIndex.getOrd(doc);
if (ord < 0) {
return compareTopMissing();
} else {
final long comparableOrd = ord << 2;
return Long.compare(topOrd, comparableOrd);
}
}
@Override
public int compareBottomMissing() {
assert bottomSlot != -1;
return Long.compare(bottomOrd, missingOrd);
}
@Override
public int compareTopMissing() {
int cmp = Long.compare(topOrd, missingOrd);
if (cmp == 0) {
return compareValues(top, missingValue);
} else {
return cmp;
}
}
@Override
public void copy(int slot, int doc) {
final int ord = termsIndex.getOrd(doc);
if (ord < 0) {
ords[slot] = missingOrd;
values[slot] = missingValue;
} else {
assert ord >= 0;
ords[slot] = ((long) ord) << 2;
if (values[slot] == null || values[slot] == missingValue) {
values[slot] = new BytesRef();
}
values[slot].copyBytes(termsIndex.lookupOrd(ord));
}
readerGen[slot] = currentReaderGen;
}
@Override
public void missing(int slot) {
ords[slot] = missingOrd;
values[slot] = missingValue;
readerGen[slot] = currentReaderGen;
}
}
// for assertions
private boolean consistentInsertedOrd(SortedDocValues termsIndex, long ord, BytesRef value) {
final int previousOrd = (int) (ord >> 2);
final int nextOrd = previousOrd + 1;
final BytesRef previous = previousOrd < 0 ? null : termsIndex.lookupOrd(previousOrd);
if ((ord & 3) == 0) { // there was an existing ord with the inserted value
assert compareValues(previous, value) == 0;
} else {
assert compareValues(previous, value) < 0;
}
if (nextOrd < termsIndex.getValueCount()) {
final BytesRef next = termsIndex.lookupOrd(nextOrd);
assert compareValues(value, next) < 0;
}
return true;
}
// find where to insert an ord in the current terms index
private long ordInCurrentReader(SortedDocValues termsIndex, BytesRef value) {
final long ord;
if (value == null) {
ord = -1 << 2;
} else {
final long docOrd = termsIndex.lookupTerm(value);
if (docOrd >= 0) {
// value exists in the current segment
ord = docOrd << 2;
} else {
// value doesn't exist, use the ord between the previous and the next term
ord = ((-2 - docOrd) << 2) + 2;
}
}
assert (ord & 1) == 0;
return ord;
}
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
termsIndex = sortMode.select(indexFieldData.load(context).getOrdinalsValues(), -1);
missingOrd = ordInCurrentReader(termsIndex, missingValue);
assert consistentInsertedOrd(termsIndex, missingOrd, missingValue);
FieldComparator<BytesRef> perSegComp = new PerSegmentComparator(termsIndex);
currentReaderGen++;
if (bottomSlot != -1) {
perSegComp.setBottom(bottomSlot);
}
if (top != null) {
perSegComp.setTopValue(top);
topOrd = ordInCurrentReader(termsIndex, top);
} else {
topOrd = missingOrd;
}
return perSegComp;
}
@Override
public void setBottom(final int bottom) {
bottomSlot = bottom;
final BytesRef bottomValue = values[bottomSlot];
if (currentReaderGen == readerGen[bottomSlot]) {
bottomOrd = ords[bottomSlot];
} else {
// insert an ord
bottomOrd = ordInCurrentReader(termsIndex, bottomValue);
if (bottomOrd == missingOrd && bottomValue != null) {
// bottomValue and missingValue and in-between the same field data values -> tie-break
// this is why we multiply ords by 4
assert missingValue != null;
final int cmp = bottomValue.compareTo(missingValue);
if (cmp < 0) {
--bottomOrd;
} else if (cmp > 0) {
++bottomOrd;
}
}
assert consistentInsertedOrd(termsIndex, bottomOrd, bottomValue);
}
}
@Override
public void setTopValue(BytesRef value) {
this.top = value;
}
@Override
public BytesRef value(int slot) {
return values[slot];
}
}

136
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java
View File

@ -1,136 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
* Sorts by field's natural Term sort order. All
* comparisons are done using BytesRef.compareTo, which is
* slow for medium to large result sets but possibly
* very fast for very small results sets.
*/
public final class BytesRefValComparator extends NestedWrappableComparator<BytesRef> {
private final IndexFieldData<?> indexFieldData;
private final MultiValueMode sortMode;
private final BytesRef missingValue;
private final BytesRef[] values;
private BytesRef bottom;
private BytesRef top;
private BinaryDocValues docTerms;
BytesRefValComparator(IndexFieldData<?> indexFieldData, int numHits, MultiValueMode sortMode, BytesRef missingValue) {
this.sortMode = sortMode;
values = new BytesRef[numHits];
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
}
@Override
public int compare(int slot1, int slot2) {
final BytesRef val1 = values[slot1];
final BytesRef val2 = values[slot2];
return compareValues(val1, val2);
}
@Override
public int compareBottom(int doc) throws IOException {
BytesRef val2 = docTerms.get(doc);
return compareValues(bottom, val2);
}
@Override
public int compareTop(int doc) throws IOException {
return top.compareTo(docTerms.get(doc));
}
@Override
public void copy(int slot, int doc) throws IOException {
BytesRef relevantValue = docTerms.get(doc);
if (relevantValue == missingValue) {
values[slot] = missingValue;
} else {
if (values[slot] == null || values[slot] == missingValue) {
values[slot] = new BytesRef();
}
values[slot].copyBytes(relevantValue);
}
}
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
final SortedBinaryDocValues docTerms = indexFieldData.load(context).getBytesValues();
this.docTerms = sortMode.select(docTerms, missingValue);
return this;
}
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
}
@Override
public void setTopValue(BytesRef top) {
this.top = top;
}
@Override
public BytesRef value(int slot) {
return values[slot];
}
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
if (val1 == null) {
if (val2 == null) {
return 0;
}
return -1;
} else if (val2 == null) {
return 1;
}
return val1.compareTo(val2);
}
@Override
public void missing(int slot) {
values[slot] = missingValue;
}
@Override
public int compareBottomMissing() {
return compareValues(bottom, missingValue);
}
@Override
public int compareTopMissing() {
return compareValues(top, missingValue);
}
}

155
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleScriptDataComparator.java
View File

@ -1,155 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SortField;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.script.SearchScript;
import java.io.IOException;
/**
*
*/
// LUCENE MONITOR: Monitor against FieldComparator.Double
public class DoubleScriptDataComparator extends NumberComparatorBase<Double> {
public static IndexFieldData.XFieldComparatorSource comparatorSource(SearchScript script) {
return new InnerSource(script);
}
private static class InnerSource extends IndexFieldData.XFieldComparatorSource {
private final SearchScript script;
private InnerSource(SearchScript script) {
this.script = script;
}
@Override
public FieldComparator<? extends Number> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
return new DoubleScriptDataComparator(numHits, script);
}
@Override
public SortField.Type reducedType() {
return SortField.Type.DOUBLE;
}
}
private final SearchScript script;
private final double[] values;
private double bottom;
public DoubleScriptDataComparator(int numHits, SearchScript script) {
this.script = script;
values = new double[numHits];
}
@Override
public FieldComparator<Double> setNextReader(AtomicReaderContext context) throws IOException {
script.setNextReader(context);
return this;
}
@Override
public void setScorer(Scorer scorer) {
script.setScorer(scorer);
}
@Override
public int compare(int slot1, int slot2) {
final double v1 = values[slot1];
final double v2 = values[slot2];
if (v1 > v2) {
return 1;
} else if (v1 < v2) {
return -1;
} else {
return 0;
}
}
@Override
public int compareBottom(int doc) {
script.setNextDocId(doc);
final double v2 = script.runAsDouble();
if (bottom > v2) {
return 1;
} else if (bottom < v2) {
return -1;
} else {
return 0;
}
}
@Override
public int compareTop(int doc) throws IOException {
script.setNextDocId(doc);
double docValue = script.runAsDouble();
return Double.compare(top, docValue);
}
@Override
public void copy(int slot, int doc) {
script.setNextDocId(doc);
values[slot] = script.runAsDouble();
}
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
}
@Override
public Double value(int slot) {
return values[slot];
}
@Override
public void add(int slot, int doc) {
script.setNextDocId(doc);
values[slot] += script.runAsDouble();
}
@Override
public void divide(int slot, int divisor) {
values[slot] /= divisor;
}
@Override
public void missing(int slot) {
values[slot] = Double.MAX_VALUE;
}
@Override
public int compareBottomMissing() {
return Double.compare(bottom, Double.MAX_VALUE);
}
@Override
public int compareTopMissing() {
return Double.compare(top, Double.MAX_VALUE);
}
}

74
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java
View File

@ -1,74 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
*/
public class DoubleValuesComparator extends DoubleValuesComparatorBase<Double> {
private final double[] values;
public DoubleValuesComparator(IndexNumericFieldData indexFieldData, double missingValue, int numHits, MultiValueMode sortMode) {
super(indexFieldData, missingValue, sortMode);
this.values = new double[numHits];
}
@Override
public int compare(int slot1, int slot2) {
final double v1 = values[slot1];
final double v2 = values[slot2];
return compare(v1, v2);
}
@Override
public void setBottom(int slot) {
this.bottom = values[slot];
}
@Override
public void copy(int slot, int doc) throws IOException {
values[slot] = readerValues.get(doc);
}
@Override
public Double value(int slot) {
return Double.valueOf(values[slot]);
}
@Override
public void add(int slot, int doc) {
values[slot] += readerValues.get(doc);
}
@Override
public void divide(int slot, int divisor) {
values[slot] /= divisor;
}
@Override
public void missing(int slot) {
values[slot] = missingValue;
}
}

79
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorBase.java
View File

@ -1,79 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldComparator;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.NumericDoubleValues;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
abstract class DoubleValuesComparatorBase<T extends Number> extends NumberComparatorBase<T> {
protected final IndexNumericFieldData indexFieldData;
protected final double missingValue;
protected double bottom;
protected NumericDoubleValues readerValues;
protected final MultiValueMode sortMode;
public DoubleValuesComparatorBase(IndexNumericFieldData indexFieldData, double missingValue, MultiValueMode sortMode) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
this.sortMode = sortMode;
}
@Override
public final int compareBottom(int doc) throws IOException {
final double v2 = readerValues.get(doc);
return compare(bottom, v2);
}
@Override
public int compareTop(int doc) throws IOException {
return compare(top.doubleValue(), readerValues.get(doc));
}
protected NumericDoubleValues getNumericDoubleValues(AtomicReaderContext context) {
SortedNumericDoubleValues readerValues = indexFieldData.load(context).getDoubleValues();
return sortMode.select(readerValues, missingValue);
}
@Override
public final FieldComparator<T> setNextReader(AtomicReaderContext context) throws IOException {
this.readerValues = getNumericDoubleValues(context);
return this;
}
@Override
public int compareBottomMissing() {
return compare(bottom, missingValue);
}
@Override
public int compareTopMissing() {
return compare(top.doubleValue(), missingValue);
}
static final int compare(double left, double right) {
return Double.compare(left, right);
}
}

46
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java
View File

@ -19,27 +19,36 @@
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldCache.Doubles;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.NumericDoubleValues;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
* Comparator source for double values.
*/
public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
private final IndexNumericFieldData indexFieldData;
private final Object missingValue;
private final MultiValueMode sortMode;
private final Nested nested;
public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
this.sortMode = sortMode;
this.nested = nested;
}
@Override
@ -47,11 +56,42 @@ public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparato
return SortField.Type.DOUBLE;
}
protected SortedNumericDoubleValues getValues(AtomicReaderContext context) {
return indexFieldData.load(context).getDoubleValues();
}
protected void setScorer(Scorer scorer) {}
@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
assert fieldname.equals(indexFieldData.getFieldNames().indexName());
assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName());
final double dMissingValue = (Double) missingObject(missingValue, reversed);
return new DoubleValuesComparator(indexFieldData, dMissingValue, numHits, sortMode);
// NOTE: it's important to pass null as a missing value in the constructor so that
// the comparator doesn't check docsWithField since we replace missing values in select()
return new FieldComparator.DoubleComparator(numHits, null, null, null) {
@Override
protected Doubles getDoubleValues(AtomicReaderContext context, String field) throws IOException {
final SortedNumericDoubleValues values = getValues(context);
final NumericDoubleValues selectedValues;
if (nested == null) {
selectedValues = sortMode.select(values, dMissingValue);
} else {
final FixedBitSet rootDocs = nested.rootDocs(context);
final FixedBitSet innerDocs = nested.innerDocs(context);
selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc());
}
return new Doubles() {
@Override
public double get(int docID) {
return selectedValues.get(docID);
}
};
}
@Override
public void setScorer(Scorer scorer) {
DoubleValuesComparatorSource.this.setScorer(scorer);
}
};
}
}

74
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparator.java
View File

@ -1,74 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
*/
public final class FloatValuesComparator extends DoubleValuesComparatorBase<Float> {
private final float[] values;
public FloatValuesComparator(IndexNumericFieldData indexFieldData, float missingValue, int numHits, MultiValueMode sortMode) {
super(indexFieldData, missingValue, sortMode);
assert indexFieldData.getNumericType().requiredBits() <= 32;
this.values = new float[numHits];
}
@Override
public int compare(int slot1, int slot2) {
final float v1 = values[slot1];
final float v2 = values[slot2];
return Float.compare(v1, v2);
}
@Override
public void setBottom(int slot) {
this.bottom = values[slot];
}
@Override
public void copy(int slot, int doc) throws IOException {
values[slot] = (float) readerValues.get(doc);
}
@Override
public Float value(int slot) {
return Float.valueOf(values[slot]);
}
@Override
public void add(int slot, int doc) {
values[slot] += (float) readerValues.get(doc);
}
@Override
public void divide(int slot, int divisor) {
values[slot] /= divisor;
}
@Override
public void missing(int slot) {
values[slot] = (float) missingValue;
}
}

35
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java
View File

@ -18,27 +18,35 @@
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldCache.Floats;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.NumericDoubleValues;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
* Comparator source for float values.
*/
public class FloatValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
private final IndexNumericFieldData indexFieldData;
private final Object missingValue;
private final MultiValueMode sortMode;
private final Nested nested;
public FloatValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
public FloatValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
this.sortMode = sortMode;
this.nested = nested;
}
@Override
@ -48,9 +56,30 @@ public class FloatValuesComparatorSource extends IndexFieldData.XFieldComparator
@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
assert fieldname.equals(indexFieldData.getFieldNames().indexName());
assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName());
final float dMissingValue = (Float) missingObject(missingValue, reversed);
return new FloatValuesComparator(indexFieldData, dMissingValue, numHits, sortMode);
// NOTE: it's important to pass null as a missing value in the constructor so that
// the comparator doesn't check docsWithField since we replace missing values in select()
return new FieldComparator.FloatComparator(numHits, null, null, null) {
@Override
protected Floats getFloatValues(AtomicReaderContext context, String field) throws IOException {
final SortedNumericDoubleValues values = indexFieldData.load(context).getDoubleValues();
final NumericDoubleValues selectedValues;
if (nested == null) {
selectedValues = sortMode.select(values, dMissingValue);
} else {
final FixedBitSet rootDocs = nested.rootDocs(context);
final FixedBitSet innerDocs = nested.innerDocs(context);
selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc());
}
return new Floats() {
@Override
public float get(int docID) {
return (float) selectedValues.get(docID);
}
};
}
};
}
}

73
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparator.java
View File

@ -1,73 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
*/
public final class LongValuesComparator extends LongValuesComparatorBase<Long> {
private final long[] values;
public LongValuesComparator(IndexNumericFieldData indexFieldData, long missingValue, int numHits, MultiValueMode sortMode) {
super(indexFieldData, missingValue, sortMode);
this.values = new long[numHits];
assert indexFieldData.getNumericType().requiredBits() <= 64;
}
@Override
public int compare(int slot1, int slot2) {
final long v1 = values[slot1];
final long v2 = values[slot2];
return Long.compare(v1, v2);
}
@Override
public void setBottom(int slot) {
this.bottom = values[slot];
}
public void copy(int slot, int doc) throws IOException {
values[slot] = readerValues.get(doc);
}
@Override
public Long value(int slot) {
return Long.valueOf(values[slot]);
}
@Override
public void add(int slot, int doc) {
values[slot] += readerValues.get(doc);
}
@Override
public void divide(int slot, int divisor) {
values[slot] /= divisor;
}
@Override
public void missing(int slot) {
values[slot] = missingValue;
}
}

72
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorBase.java
View File

@ -1,72 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.search.FieldComparator;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
abstract class LongValuesComparatorBase<T extends Number> extends NumberComparatorBase<T> {
protected final IndexNumericFieldData indexFieldData;
protected final long missingValue;
protected long bottom;
protected NumericDocValues readerValues;
protected final MultiValueMode sortMode;
public LongValuesComparatorBase(IndexNumericFieldData indexFieldData, long missingValue, MultiValueMode sortMode) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
this.sortMode = sortMode;
}
@Override
public final int compareBottom(int doc) throws IOException {
long v2 = readerValues.get(doc);
return Long.compare(bottom, v2);
}
@Override
public int compareTop(int doc) throws IOException {
return Long.compare(top.longValue(), readerValues.get(doc));
}
@Override
public final FieldComparator<T> setNextReader(AtomicReaderContext context) throws IOException {
SortedNumericDocValues readerValues = indexFieldData.load(context).getLongValues();
this.readerValues = sortMode.select(readerValues, missingValue);
return this;
}
@Override
public int compareBottomMissing() {
return Long.compare(bottom, missingValue);
}
@Override
public int compareTopMissing() {
return Long.compare(top.longValue(), missingValue);
}
}

38
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java
View File

@ -18,8 +18,13 @@
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.search.FieldCache.Longs;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
@ -28,17 +33,20 @@ import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
/**
* Comparator source for long values.
*/
public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
private final IndexNumericFieldData indexFieldData;
private final Object missingValue;
private final MultiValueMode sortMode;
private final Nested nested;
public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode) {
public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
this.sortMode = sortMode;
this.nested = nested;
}
@Override
@ -48,9 +56,31 @@ public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorS
@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
assert fieldname.equals(indexFieldData.getFieldNames().indexName());
assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName());
final long dMissingValue = (Long) missingObject(missingValue, reversed);
return new LongValuesComparator(indexFieldData, dMissingValue, numHits, sortMode);
final Long dMissingValue = (Long) missingObject(missingValue, reversed);
// NOTE: it's important to pass null as a missing value in the constructor so that
// the comparator doesn't check docsWithField since we replace missing values in select()
return new FieldComparator.LongComparator(numHits, null, null, null) {
@Override
protected Longs getLongValues(AtomicReaderContext context, String field) throws IOException {
final SortedNumericDocValues values = indexFieldData.load(context).getLongValues();
final NumericDocValues selectedValues;
if (nested == null) {
selectedValues = sortMode.select(values, dMissingValue);
} else {
final FixedBitSet rootDocs = nested.rootDocs(context);
final FixedBitSet innerDocs = nested.innerDocs(context);
selectedValues = sortMode.select(values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc());
}
return new Longs() {
@Override
public long get(int docID) {
return selectedValues.get(docID);
}
};
}
};
}
}

49
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NestedWrappableComparator.java
View File

@ -1,49 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.search.FieldComparator;
/** Base comparator which allows for nested sorting. */
public abstract class NestedWrappableComparator<T> extends FieldComparator<T> {
/**
* Assigns the underlying missing value to the specified slot, if the actual implementation supports missing value.
*
* @param slot The slot to assign the the missing value to.
*/
public abstract void missing(int slot);
/**
* Compares the missing value to the bottom.
*
* @return any N < 0 if the bottom value is not competitive with the missing value, any N > 0 if the
* bottom value is competitive with the missing value and 0 if they are equal.
*/
public abstract int compareBottomMissing();
/**
* Compares the missing value to the top.
*
* @return any N < 0 if the tope is not competitive with the missing value, any N > 0 if the top is competitive
* with the top and 0 if they are equal.
*/
public abstract int compareTopMissing();
}

50
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NumberComparatorBase.java
View File

@ -1,50 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
/**
* Base FieldComparator class for number fields.
*/
// This is right now only used for sorting number based fields inside nested objects
public abstract class NumberComparatorBase<T> extends NestedWrappableComparator<T> {
protected T top;
/**
* Adds numeric value at the specified doc to the specified slot.
*
* @param slot The specified slot
* @param doc The specified doc
*/
public abstract void add(int slot, int doc);
/**
* Divides the value at the specified slot with the specified divisor.
*
* @param slot The specified slot
* @param divisor The specified divisor
*/
public abstract void divide(int slot, int divisor);
@Override
public void setTopValue(T top) {
this.top = top;
}
}

154
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/StringScriptDataComparator.java
View File

@ -1,154 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.script.SearchScript;
import java.io.IOException;
/**
*
*/
public class StringScriptDataComparator extends FieldComparator<BytesRef> {
private BytesRef top;
public static IndexFieldData.XFieldComparatorSource comparatorSource(SearchScript script) {
return new InnerSource(script);
}
private static class InnerSource extends IndexFieldData.XFieldComparatorSource {
private final SearchScript script;
private InnerSource(SearchScript script) {
this.script = script;
}
@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
return new StringScriptDataComparator(numHits, script);
}
@Override
public SortField.Type reducedType() {
return SortField.Type.STRING;
}
}
private final SearchScript script;
private BytesRef[] values; // TODO maybe we can preallocate or use a sentinel to prevent the conditionals in compare
private BytesRef bottom;
private final BytesRef spare = new BytesRef();
private int spareDoc = -1;
public StringScriptDataComparator(int numHits, SearchScript script) {
this.script = script;
values = new BytesRef[numHits];
}
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
script.setNextReader(context);
spareDoc = -1;
return this;
}
@Override
public void setScorer(Scorer scorer) {
script.setScorer(scorer);
}
@Override
public int compare(int slot1, int slot2) {
final BytesRef val1 = values[slot1];
final BytesRef val2 = values[slot2];
if (val1 == null) {
if (val2 == null) {
return 0;
}
return -1;
} else if (val2 == null) {
return 1;
}
return val1.compareTo(val2);
}
@Override
public int compareBottom(int doc) {
if (bottom == null) {
return -1;
}
setSpare(doc);
return bottom.compareTo(spare);
}
@Override
public int compareTop(int doc) throws IOException {
script.setNextDocId(doc);
setSpare(doc);
return top.compareTo(spare);
}
private void setSpare(int doc) {
if (spareDoc == doc) {
return;
}
script.setNextDocId(doc);
spare.copyChars(script.run().toString());
spareDoc = doc;
}
@Override
public void copy(int slot, int doc) {
setSpare(doc);
if (values[slot] == null) {
values[slot] = BytesRef.deepCopyOf(spare);
} else {
values[slot].copyBytes(spare);
}
}
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
}
@Override
public void setTopValue(BytesRef top) {
this.top = top;
}
@Override
public BytesRef value(int slot) {
return values[slot];
}
}

8
src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
View File

@ -25,10 +25,8 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.search.MultiValueMode;
@ -74,7 +72,7 @@ public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponen
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
throw new UnsupportedOperationException("no global ordinals sorting yet");
}

8
src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
View File

@ -28,10 +28,8 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.search.MultiValueMode;
@ -80,7 +78,7 @@ abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData<Ato
}
@Override
public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
throw new ElasticsearchIllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");
}

10
src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
View File

@ -26,10 +26,8 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper.Names;
@ -57,8 +55,8 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
}
@Override

5
src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVIndexFieldData.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.index.AtomicReaderContext;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.search.MultiValueMode;
@ -44,7 +45,7 @@ public class BinaryDVIndexFieldData extends DocValuesIndexFieldData implements I
}
@Override
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) {
return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
}
}

14
src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVNumericIndexFieldData.java
View File

@ -31,10 +31,8 @@ import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.common.util.ByteUtils;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
@ -53,15 +51,15 @@ public class BinaryDVNumericIndexFieldData extends DocValuesIndexFieldData imple
this.numericType = numericType;
}
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(final Object missingValue, final MultiValueMode sortMode) {
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(final Object missingValue, final MultiValueMode sortMode, Nested nested) {
switch (numericType) {
case FLOAT:
return new FloatValuesComparatorSource(this, missingValue, sortMode);
return new FloatValuesComparatorSource(this, missingValue, sortMode, nested);
case DOUBLE:
return new DoubleValuesComparatorSource(this, missingValue, sortMode);
return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested);
default:
assert !numericType.isFloatingPoint();
return new LongValuesComparatorSource(this, missingValue, sortMode);
return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
}
}

3
src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVIndexFieldData.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMapper.Names;
@ -44,7 +45,7 @@ public class BytesBinaryDVIndexFieldData extends DocValuesIndexFieldData impleme
}
@Override
public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
throw new ElasticsearchIllegalArgumentException("can't sort on binary field");
}

8
src/main/java/org/elasticsearch/index/fielddata/plain/DisabledIndexFieldData.java
View File

@ -23,10 +23,8 @@ import org.apache.lucene.index.AtomicReaderContext;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.index.mapper.MapperService;
@ -59,7 +57,7 @@ public final class DisabledIndexFieldData extends AbstractIndexFieldData<AtomicF
}
@Override
public IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) {
public IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
throw fail();
}

5
src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
View File

@ -27,6 +27,7 @@ import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.DoubleArray;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
@ -155,8 +156,8 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
return new DoubleValuesComparatorSource(this, missingValue, sortMode);
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested);
}
private static SortedNumericDoubleValues withOrdinals(Ordinals ordinals, final DoubleArray values, int maxDoc) {

5
src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.FloatArray;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
@ -153,8 +154,8 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<AtomicNumer
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
return new FloatValuesComparatorSource(this, missingValue, sortMode);
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
return new FloatValuesComparatorSource(this, missingValue, sortMode, nested);
}
private static SortedNumericDoubleValues withOrdinals(Ordinals ordinals, final FloatArray values, int maxDoc) {

3
src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointBinaryDVIndexFieldData.java
View File

@ -27,6 +27,7 @@ import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.index.mapper.MapperService;
@ -42,7 +43,7 @@ public class GeoPointBinaryDVIndexFieldData extends DocValuesIndexFieldData impl
}
@Override
public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
throw new ElasticsearchIllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");
}

5
src/main/java/org/elasticsearch/index/fielddata/plain/NumericDVIndexFieldData.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.util.Bits;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
import org.elasticsearch.index.mapper.FieldMapper.Names;
@ -62,8 +63,8 @@ public class NumericDVIndexFieldData extends DocValuesIndexFieldData implements
}
@Override
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) {
return new LongValuesComparatorSource(this, missingValue, sortMode);
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
}
@Override

5
src/main/java/org/elasticsearch/index/fielddata/plain/PackedArrayIndexFieldData.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.common.breaker.MemoryCircuitBreaker;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
@ -343,8 +344,8 @@ public class PackedArrayIndexFieldData extends AbstractIndexFieldData<AtomicNume
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
return new LongValuesComparatorSource(this, missingValue, sortMode);
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
}
/**

2
src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildAtomicFieldData.java
View File

@ -63,7 +63,7 @@ public class ParentChildAtomicFieldData extends AbstractAtomicParentChildFieldDa
public SortedDocValues getOrdinalsValues(String type) {
AtomicOrdinalsFieldData atomicFieldData = typeToIds.get(type);
if (atomicFieldData != null) {
return MultiValueMode.MIN.select(atomicFieldData.getOrdinalsValues(), -1);
return MultiValueMode.MIN.select(atomicFieldData.getOrdinalsValues());
} else {
return DocValues.emptySorted();
}

8
src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
View File

@ -38,6 +38,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
@ -79,8 +80,8 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode) {
return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
}
@Override
@ -408,8 +409,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
}
@Override
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue,
MultiValueMode sortMode) {
public XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
throw new UnsupportedOperationException("No sorting on global ords");
}

5
src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.elasticsearch.index.mapper.FieldMapper.Names;
@ -44,8 +45,8 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i
this.breakerService = breakerService;
}
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) {
return new BytesRefFieldComparatorSource((IndexFieldData<?>) this, missingValue, sortMode);
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
return new BytesRefFieldComparatorSource((IndexFieldData<?>) this, missingValue, sortMode, nested);
}
@Override

500
src/main/java/org/elasticsearch/index/search/nested/NestedFieldComparatorSource.java
View File

@ -1,500 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.search.nested;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.common.lucene.docset.DocIdSets;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.fieldcomparator.NestedWrappableComparator;
import org.elasticsearch.index.fielddata.fieldcomparator.NumberComparatorBase;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
import java.util.Locale;
/**
*/
public class NestedFieldComparatorSource extends IndexFieldData.XFieldComparatorSource {
private final MultiValueMode sortMode;
private final IndexFieldData.XFieldComparatorSource wrappedSource;
private final Filter rootDocumentsFilter;
private final Filter innerDocumentsFilter;
public NestedFieldComparatorSource(MultiValueMode sortMode, IndexFieldData.XFieldComparatorSource wrappedSource, Filter rootDocumentsFilter, Filter innerDocumentsFilter) {
this.sortMode = sortMode;
this.wrappedSource = wrappedSource;
this.rootDocumentsFilter = rootDocumentsFilter;
this.innerDocumentsFilter = innerDocumentsFilter;
}
@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
// +1: have one spare slot for value comparison between inner documents.
FieldComparator wrappedComparator = wrappedSource.newComparator(fieldname, numHits + 1, sortPos, reversed);
switch (sortMode) {
case MAX:
return new NestedFieldComparator.Highest(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
case MIN:
return new NestedFieldComparator.Lowest(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
case SUM:
return new NestedFieldComparator.Sum((NumberComparatorBase<?>) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
case AVG:
return new NestedFieldComparator.Avg((NumberComparatorBase<?>) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
default:
throw new ElasticsearchIllegalArgumentException(
String.format(Locale.ROOT, "Unsupported sort_mode[%s] for nested type", sortMode)
);
}
}
@Override
public SortField.Type reducedType() {
return wrappedSource.reducedType();
}
}
abstract class NestedFieldComparator extends FieldComparator {
final Filter rootDocumentsFilter;
final Filter innerDocumentsFilter;
final int spareSlot;
FieldComparator wrappedComparator;
FixedBitSet rootDocuments;
FixedBitSet innerDocuments;
int bottomSlot;
Object top;
NestedFieldComparator(FieldComparator wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) {
this.wrappedComparator = wrappedComparator;
this.rootDocumentsFilter = rootDocumentsFilter;
this.innerDocumentsFilter = innerDocumentsFilter;
this.spareSlot = spareSlot;
}
@Override
public final int compare(int slot1, int slot2) {
return wrappedComparator.compare(slot1, slot2);
}
@Override
public final void setBottom(int slot) {
wrappedComparator.setBottom(slot);
this.bottomSlot = slot;
}
@Override
public FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
DocIdSet innerDocuments = innerDocumentsFilter.getDocIdSet(context, null);
if (DocIdSets.isEmpty(innerDocuments)) {
this.innerDocuments = null;
} else if (innerDocuments instanceof FixedBitSet) {
this.innerDocuments = (FixedBitSet) innerDocuments;
} else {
this.innerDocuments = DocIdSets.toFixedBitSet(innerDocuments.iterator(), context.reader().maxDoc());
}
DocIdSet rootDocuments = rootDocumentsFilter.getDocIdSet(context, null);
if (DocIdSets.isEmpty(rootDocuments)) {
this.rootDocuments = null;
} else if (rootDocuments instanceof FixedBitSet) {
this.rootDocuments = (FixedBitSet) rootDocuments;
} else {
this.rootDocuments = DocIdSets.toFixedBitSet(rootDocuments.iterator(), context.reader().maxDoc());
}
wrappedComparator = wrappedComparator.setNextReader(context);
return this;
}
@Override
public final Object value(int slot) {
return wrappedComparator.value(slot);
}
@Override
public void setTopValue(Object top) {
this.top = top;
wrappedComparator.setTopValue(top);
}
final static class Lowest extends NestedFieldComparator {
Lowest(FieldComparator wrappedComparator, Filter parentFilter, Filter childFilter, int spareSlot) {
super(wrappedComparator, parentFilter, childFilter, spareSlot);
}
@Override
public int compareBottom(int rootDoc) throws IOException {
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
return compareBottomMissing(wrappedComparator);
}
// We need to copy the lowest value from all nested docs into slot.
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return compareBottomMissing(wrappedComparator);
}
// We only need to emit a single cmp value for any matching nested doc
int cmp = wrappedComparator.compareBottom(nestedDoc);
if (cmp > 0) {
return cmp;
}
while (true) {
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return cmp;
}
int cmp1 = wrappedComparator.compareBottom(nestedDoc);
if (cmp1 > 0) {
return cmp1;
} else {
if (cmp1 == 0) {
cmp = 0;
}
}
}
}
@Override
public void copy(int slot, int rootDoc) throws IOException {
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
copyMissing(wrappedComparator, slot);
return;
}
// We need to copy the lowest value from all nested docs into slot.
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
copyMissing(wrappedComparator, slot);
return;
}
wrappedComparator.copy(slot, nestedDoc);
while (true) {
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return;
}
wrappedComparator.copy(spareSlot, nestedDoc);
if (wrappedComparator.compare(spareSlot, slot) < 0) {
wrappedComparator.copy(slot, nestedDoc);
}
}
}
@Override
public int compareTop(int rootDoc) throws IOException {
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
return compareTopMissing(wrappedComparator);
}
// We need to copy the lowest value from all nested docs into slot.
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return compareTopMissing(wrappedComparator);
}
// We only need to emit a single cmp value for any matching nested doc
@SuppressWarnings("unchecked")
int cmp = wrappedComparator.compareTop(nestedDoc);
if (cmp > 0) {
return cmp;
}
while (true) {
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return cmp;
}
@SuppressWarnings("unchecked")
int cmp1 = wrappedComparator.compareTop(nestedDoc);
if (cmp1 > 0) {
return cmp1;
} else {
if (cmp1 == 0) {
cmp = 0;
}
}
}
}
}
final static class Highest extends NestedFieldComparator {
Highest(FieldComparator wrappedComparator, Filter parentFilter, Filter childFilter, int spareSlot) {
super(wrappedComparator, parentFilter, childFilter, spareSlot);
}
@Override
public int compareBottom(int rootDoc) throws IOException {
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
return compareBottomMissing(wrappedComparator);
}
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return compareBottomMissing(wrappedComparator);
}
int cmp = wrappedComparator.compareBottom(nestedDoc);
if (cmp < 0) {
return cmp;
}
while (true) {
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return cmp;
}
int cmp1 = wrappedComparator.compareBottom(nestedDoc);
if (cmp1 < 0) {
return cmp1;
} else if (cmp1 == 0) {
cmp = 0;
}
}
}
@Override
public void copy(int slot, int rootDoc) throws IOException {
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
copyMissing(wrappedComparator, slot);
return;
}
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
copyMissing(wrappedComparator, slot);
return;
}
wrappedComparator.copy(slot, nestedDoc);
while (true) {
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return;
}
wrappedComparator.copy(spareSlot, nestedDoc);
if (wrappedComparator.compare(spareSlot, slot) > 0) {
wrappedComparator.copy(slot, nestedDoc);
}
}
}
@Override
public int compareTop(int rootDoc) throws IOException {
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
return compareTopMissing(wrappedComparator);
}
int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return compareTopMissing(wrappedComparator);
}
@SuppressWarnings("unchecked")
int cmp = wrappedComparator.compareTop(nestedDoc);
if (cmp < 0) {
return cmp;
}
while (true) {
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return cmp;
}
@SuppressWarnings("unchecked")
int cmp1 = wrappedComparator.compareTop(nestedDoc);
if (cmp1 < 0) {
return cmp1;
} else if (cmp1 == 0) {
cmp = 0;
}
}
}
}
static abstract class NumericNestedFieldComparatorBase extends NestedFieldComparator {
protected NumberComparatorBase numberComparator;
NumericNestedFieldComparatorBase(NumberComparatorBase wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) {
super(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, spareSlot);
this.numberComparator = wrappedComparator;
}
@Override
public final int compareBottom(int rootDoc) throws IOException {
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
return compareBottomMissing(wrappedComparator);
}
final int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return compareBottomMissing(wrappedComparator);
}
int counter = 1;
wrappedComparator.copy(spareSlot, nestedDoc);
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
while (nestedDoc > prevRootDoc && nestedDoc < rootDoc) {
onNested(spareSlot, nestedDoc);
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
counter++;
}
afterNested(spareSlot, counter);
return compare(bottomSlot, spareSlot);
}
@Override
public final void copy(int slot, int rootDoc) throws IOException {
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
copyMissing(wrappedComparator, slot);
return;
}
final int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
copyMissing(wrappedComparator, slot);
return;
}
int counter = 1;
wrappedComparator.copy(slot, nestedDoc);
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
while (nestedDoc > prevRootDoc && nestedDoc < rootDoc) {
onNested(slot, nestedDoc);
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
counter++;
}
afterNested(slot, counter);
}
@Override
@SuppressWarnings("unchecked")
public int compareTop(int rootDoc) throws IOException {
if (rootDoc == 0 || rootDocuments == null || innerDocuments == null) {
return compareTopMissing(wrappedComparator);
}
final int prevRootDoc = rootDocuments.prevSetBit(rootDoc - 1);
int nestedDoc = innerDocuments.nextSetBit(prevRootDoc + 1);
if (nestedDoc >= rootDoc || nestedDoc == -1) {
return compareTopMissing(wrappedComparator);
}
int counter = 1;
wrappedComparator.copy(spareSlot, nestedDoc);
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
while (nestedDoc > prevRootDoc && nestedDoc < rootDoc) {
onNested(spareSlot, nestedDoc);
nestedDoc = innerDocuments.nextSetBit(nestedDoc + 1);
counter++;
}
afterNested(spareSlot, counter);
return wrappedComparator.compareValues(wrappedComparator.value(spareSlot), top);
}
protected abstract void onNested(int slot, int nestedDoc);
protected abstract void afterNested(int slot, int count);
@Override
public final FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
super.setNextReader(context);
numberComparator = (NumberComparatorBase) super.wrappedComparator;
return this;
}
}
final static class Sum extends NumericNestedFieldComparatorBase {
Sum(NumberComparatorBase wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) {
super(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, spareSlot);
}
@Override
protected void onNested(int slot, int nestedDoc) {
numberComparator.add(slot, nestedDoc);
}
@Override
protected void afterNested(int slot, int count) {
}
}
final static class Avg extends NumericNestedFieldComparatorBase {
Avg(NumberComparatorBase wrappedComparator, Filter rootDocumentsFilter, Filter innerDocumentsFilter, int spareSlot) {
super(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, spareSlot);
}
@Override
protected void onNested(int slot, int nestedDoc) {
numberComparator.add(slot, nestedDoc);
}
@Override
protected void afterNested(int slot, int count) {
numberComparator.divide(slot, count);
}
}
static final void copyMissing(FieldComparator<?> comparator, int slot) {
if (comparator instanceof NestedWrappableComparator<?>) {
((NestedWrappableComparator<?>) comparator).missing(slot);
}
}
static final int compareBottomMissing(FieldComparator<?> comparator) {
if (comparator instanceof NestedWrappableComparator<?>) {
return ((NestedWrappableComparator<?>) comparator).compareBottomMissing();
} else {
return 0;
}
}
@SuppressWarnings("unchecked")
static final int compareTopMissing(FieldComparator<?> comparator) {
if (comparator instanceof NestedWrappableComparator) {
return ((NestedWrappableComparator) comparator).compareTopMissing();
} else {
return 0;
}
}
}

295
src/main/java/org/elasticsearch/search/MultiValueMode.java
View File

@ -23,6 +23,7 @@ package org.elasticsearch.search;
import org.apache.lucene.index.*;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.index.fielddata.FieldData;
import org.elasticsearch.index.fielddata.NumericDoubleValues;
@ -91,7 +92,13 @@ public enum MultiValueMode {
*/
@Override
public long reduce(long a, int numValues) {
return Math.round(a / Math.max(1.0, numValues));
if (numValues <= 1) {
// without this the average might be different from the original
// values on a single-valued field due to the precision loss of the double
return a;
} else {
return Math.round(a / Math.max(1.0, numValues));
}
}
},
@ -115,6 +122,16 @@ public enum MultiValueMode {
return Math.min(a, b);
}
@Override
public int applyOrd(int ord1, int ord2) {
return Math.min(ord1, ord2);
}
@Override
public BytesRef apply(BytesRef a, BytesRef b) {
return a.compareTo(b) <= 0 ? a : b;
}
/**
* Returns {@link Double#POSITIVE_INFINITY}
*/
@ -159,11 +176,11 @@ public enum MultiValueMode {
}
@Override
protected int pick(RandomAccessOrds values, int missingOrd) {
protected int pick(RandomAccessOrds values) {
if (values.cardinality() > 0) {
return (int) values.ordAt(0);
} else {
return missingOrd;
return -1;
}
}
},
@ -188,6 +205,16 @@ public enum MultiValueMode {
return Math.max(a, b);
}
@Override
public int applyOrd(int ord1, int ord2) {
return Math.max(ord1, ord2);
}
@Override
public BytesRef apply(BytesRef a, BytesRef b) {
return a.compareTo(b) > 0 ? a : b;
}
/**
* Returns {@link Double#NEGATIVE_INFINITY}
*/
@ -236,12 +263,12 @@ public enum MultiValueMode {
}
@Override
protected int pick(RandomAccessOrds values, int missingOrd) {
protected int pick(RandomAccessOrds values) {
final int count = values.cardinality();
if (count > 0) {
return (int) values.ordAt(count - 1);
} else {
return missingOrd;
return -1;
}
}
};
@ -274,6 +301,14 @@ public enum MultiValueMode {
*/
public abstract long apply(long a, long b);
public int applyOrd(int ord1, int ord2) {
throw new UnsupportedOperationException();
}
public BytesRef apply(BytesRef a, BytesRef b) {
throw new UnsupportedOperationException();
}
/**
* Returns an initial value for the sort mode that is guaranteed to have no impact if passed
* to {@link #apply(double, double)}. This value should be used as the initial value if the
@ -359,6 +394,11 @@ public enum MultiValueMode {
}
}
/**
* Return a {@link NumericDocValues} instance that can be used to sort documents
* with this mode and the provided values. When a document has no value,
* <code>missingValue</code> is returned.
*/
public NumericDocValues select(final SortedNumericDocValues values, final long missingValue) {
final NumericDocValues singleton = DocValues.unwrapSingleton(values);
if (singleton != null) {
@ -388,6 +428,51 @@ public enum MultiValueMode {
}
}
/**
* Return a {@link NumericDocValues} instance that can be used to sort root documents
* with this mode, the provided values and filters for root/inner documents.
*
* For every root document, the values of its inner documents will be aggregated.
* If none of the inner documents has a value, then <code>missingValue</code> is returned.
*
* NOTE: Calling the returned instance on docs that are not root docs is illegal
*/
public NumericDocValues select(final SortedNumericDocValues values, final long missingValue, final FixedBitSet rootDocs, final FixedBitSet innerDocs, int maxDoc) {
if (rootDocs == null || innerDocs == null) {
return select(FieldData.emptySortedNumeric(maxDoc), missingValue);
}
return new NumericDocValues() {
@Override
public long get(int rootDoc) {
assert rootDocs.get(rootDoc) : "can only sort root documents";
if (rootDoc == 0) {
return missingValue;
}
final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1);
long accumulated = startLong();
int numValues = 0;
for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) {
values.setDocument(doc);
final int count = values.count();
for (int i = 0; i < count; ++i) {
final long value = values.valueAt(i);
accumulated = apply(accumulated, value);
}
numValues += count;
}
return numValues == 0
? missingValue
: reduce(accumulated, numValues);
}
};
}
protected double pick(SortedNumericDoubleValues values, double missingValue) {
final int count = values.count();
if (count == 0) {
@ -401,6 +486,11 @@ public enum MultiValueMode {
}
}
/**
* Return a {@link NumericDoubleValues} instance that can be used to sort documents
* with this mode and the provided values. When a document has no value,
* <code>missingValue</code> is returned.
*/
public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue) {
final NumericDoubleValues singleton = FieldData.unwrapSingleton(values);
if (singleton != null) {
@ -430,15 +520,65 @@ public enum MultiValueMode {
}
}
/**
* Return a {@link NumericDoubleValues} instance that can be used to sort root documents
* with this mode, the provided values and filters for root/inner documents.
*
* For every root document, the values of its inner documents will be aggregated.
* If none of the inner documents has a value, then <code>missingValue</code> is returned.
*
* NOTE: Calling the returned instance on docs that are not root docs is illegal
*/
public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final FixedBitSet rootDocs, final FixedBitSet innerDocs, int maxDoc) {
if (rootDocs == null || innerDocs == null) {
return select(FieldData.emptySortedNumericDoubles(maxDoc), missingValue);
}
return new NumericDoubleValues() {
@Override
public double get(int rootDoc) {
assert rootDocs.get(rootDoc) : "can only sort root documents";
if (rootDoc == 0) {
return missingValue;
}
final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1);
double accumulated = startDouble();
int numValues = 0;
for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) {
values.setDocument(doc);
final int count = values.count();
for (int i = 0; i < count; ++i) {
final double value = values.valueAt(i);
accumulated = apply(accumulated, value);
}
numValues += count;
}
return numValues == 0
? missingValue
: reduce(accumulated, numValues);
}
};
}
protected BytesRef pick(SortedBinaryDocValues values, BytesRef missingValue) {
throw new ElasticsearchIllegalArgumentException("Unsupported sort mode: " + this);
}
/**
* Return a {@link BinaryDocValues} instance that can be used to sort documents
* with this mode and the provided values. When a document has no value,
* <code>missingValue</code> is returned.
*/
public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue) {
final BinaryDocValues singleton = FieldData.unwrapSingleton(values);
if (singleton != null) {
final Bits docsWithField = FieldData.unwrapSingletonBits(values);
if (docsWithField == null || missingValue == null || missingValue.length == 0) {
if (docsWithField == null) {
return singleton;
} else {
return new BinaryDocValues() {
@ -463,47 +603,85 @@ public enum MultiValueMode {
}
}
protected int pick(RandomAccessOrds values, int missingOrd) {
/**
* Return a {@link BinaryDocValues} instance that can be used to sort root documents
* with this mode, the provided values and filters for root/inner documents.
*
* For every root document, the values of its inner documents will be aggregated.
* If none of the inner documents has a value, then <code>missingValue</code> is returned.
*
* NOTE: Calling the returned instance on docs that are not root docs is illegal
*/
public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final FixedBitSet rootDocs, final FixedBitSet innerDocs, int maxDoc) {
if (rootDocs == null || innerDocs == null) {
return select(FieldData.emptySortedBinary(maxDoc), missingValue);
}
final BinaryDocValues selectedValues = select(values, new BytesRef());
final Bits docsWithValue;
if (FieldData.unwrapSingleton(values) != null) {
docsWithValue = FieldData.unwrapSingletonBits(values);
} else {
docsWithValue = FieldData.docsWithValue(values, maxDoc);
}
return new BinaryDocValues() {
final BytesRef spare = new BytesRef();
@Override
public BytesRef get(int rootDoc) {
assert rootDocs.get(rootDoc) : "can only sort root documents";
if (rootDoc == 0) {
return missingValue;
}
final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1);
BytesRef accumulated = null;
for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) {
values.setDocument(doc);
final BytesRef innerValue = selectedValues.get(doc);
if (innerValue.length > 0 || docsWithValue == null || docsWithValue.get(doc)) {
if (accumulated == null) {
spare.copyBytes(innerValue);
accumulated = spare;
} else {
final BytesRef applied = apply(accumulated, innerValue);
if (applied == innerValue) {
accumulated.copyBytes(innerValue);
}
}
}
}
return accumulated == null ? missingValue : accumulated;
}
};
}
protected int pick(RandomAccessOrds values) {
throw new ElasticsearchIllegalArgumentException("Unsupported sort mode: " + this);
}
public SortedDocValues select(final RandomAccessOrds values, final int missingOrd) {
/**
* Return a {@link SortedDocValues} instance that can be used to sort documents
* with this mode and the provided values.
*/
public SortedDocValues select(final RandomAccessOrds values) {
if (values.getValueCount() >= Integer.MAX_VALUE) {
throw new UnsupportedOperationException("fields containing more than " + (Integer.MAX_VALUE-1) + " unique terms are unsupported");
}
final SortedDocValues singleton = DocValues.unwrapSingleton(values);
if (singleton != null) {
if (missingOrd == -1) {
return singleton;
} else {
return new SortedDocValues() {
@Override
public int getOrd(int docID) {
final int ord = singleton.getOrd(docID);
if (ord == -1) {
return missingOrd;
}
return ord;
}
@Override
public BytesRef lookupOrd(int ord) {
return values.lookupOrd(ord);
}
@Override
public int getValueCount() {
return (int) values.getValueCount();
}
};
}
return singleton;
} else {
return new SortedDocValues() {
@Override
public int getOrd(int docID) {
values.setDocument(docID);
return pick(values, missingOrd);
return pick(values);
}
@Override
@ -519,4 +697,55 @@ public enum MultiValueMode {
}
}
/**
* Return a {@link SortedDocValues} instance that can be used to sort root documents
* with this mode, the provided values and filters for root/inner documents.
*
* For every root document, the values of its inner documents will be aggregated.
*
* NOTE: Calling the returned instance on docs that are not root docs is illegal
*/
public SortedDocValues select(final RandomAccessOrds values, final FixedBitSet rootDocs, final FixedBitSet innerDocs) {
if (rootDocs == null || innerDocs == null) {
return select((RandomAccessOrds) DocValues.emptySortedSet());
}
final SortedDocValues selectedValues = select(values);
return new SortedDocValues() {
@Override
public BytesRef lookupOrd(int ord) {
return selectedValues.lookupOrd(ord);
}
@Override
public int getValueCount() {
return selectedValues.getValueCount();
}
@Override
public int getOrd(int rootDoc) {
assert rootDocs.get(rootDoc) : "can only sort root documents";
if (rootDoc == 0) {
return -1;
}
final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
final int firstNestedDoc = innerDocs.nextSetBit(prevRootDoc + 1);
int ord = -1;
for (int doc = firstNestedDoc; doc != -1 && doc < rootDoc; doc = innerDocs.nextSetBit(doc + 1)) {
final int innerOrd = selectedValues.getOrd(doc);
if (innerOrd != -1) {
if (ord == -1) {
ord = innerOrd;
} else {
ord = applyOrd(ord, innerOrd);
}
}
}
return ord;
}
};
}
}

68
src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java
View File

@ -20,10 +20,11 @@
package org.elasticsearch.search.sort;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldCache.Doubles;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortField.Type;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance;
@ -32,12 +33,11 @@ import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparator;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.ObjectMappers;
import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.query.ParsedFilter;
import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
import org.elasticsearch.index.search.nested.NonNestedDocsFilter;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.search.internal.SearchContext;
@ -127,27 +127,6 @@ public class GeoDistanceSortParser implements SortParser {
final MultiValueMode finalSortMode = sortMode; // final reference for use in the anonymous class
final IndexGeoPointFieldData geoIndexFieldData = context.fieldData().getForField(mapper);
final FixedSourceDistance distance = geoDistance.fixedSourceDistance(point.lat(), point.lon(), unit);
IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() {
@Override
public Type reducedType() {
return Type.DOUBLE;
}
@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
return new DoubleValuesComparator(null, Double.MAX_VALUE, numHits, finalSortMode) {
@Override
protected NumericDoubleValues getNumericDoubleValues(AtomicReaderContext context) {
final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues();
final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(distance, geoPointValues);
return sortMode.select(distanceValues, Double.MAX_VALUE);
}
};
}
};
ObjectMapper objectMapper;
if (nestedPath != null) {
ObjectMappers objectMappers = context.mapperService().objectMapper(nestedPath);
@ -161,6 +140,7 @@ public class GeoDistanceSortParser implements SortParser {
} else {
objectMapper = context.mapperService().resolveClosestNestedObjectMapper(fieldName);
}
final Nested nested;
if (objectMapper != null && objectMapper.nested().isNested()) {
Filter rootDocumentsFilter = context.filterCache().cache(NonNestedDocsFilter.INSTANCE);
Filter innerDocumentsFilter;
@ -169,11 +149,45 @@ public class GeoDistanceSortParser implements SortParser {
} else {
innerDocumentsFilter = context.filterCache().cache(objectMapper.nestedTypeFilter());
}
geoDistanceComparatorSource = new NestedFieldComparatorSource(
sortMode, geoDistanceComparatorSource, rootDocumentsFilter, innerDocumentsFilter
);
nested = new Nested(rootDocumentsFilter, innerDocumentsFilter);
} else {
nested = null;
}
IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() {
@Override
public SortField.Type reducedType() {
return SortField.Type.DOUBLE;
}
@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
return new FieldComparator.DoubleComparator(numHits, null, null, null) {
@Override
protected Doubles getDoubleValues(AtomicReaderContext context, String field) throws IOException {
final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues();
final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(distance, geoPointValues);
final NumericDoubleValues selectedValues;
if (nested == null) {
selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE);
} else {
final FixedBitSet rootDocs = nested.rootDocs(context);
final FixedBitSet innerDocs = nested.innerDocs(context);
selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE, rootDocs, innerDocs, context.reader().maxDoc());
}
return new Doubles() {
@Override
public double get(int docID) {
return selectedValues.get(docID);
}
};
}
};
}
};
return new SortField(fieldName, geoDistanceComparatorSource, reverse);
}
}

83
src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java
View File

@ -19,21 +19,25 @@
package org.elasticsearch.search.sort;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleScriptDataComparator;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.fielddata.fieldcomparator.StringScriptDataComparator;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
import org.elasticsearch.index.mapper.ObjectMappers;
import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.query.ParsedFilter;
import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
import org.elasticsearch.index.search.nested.NonNestedDocsFilter;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.search.SearchParseException;
import org.elasticsearch.search.internal.SearchContext;
@ -44,6 +48,9 @@ import java.util.Map;
*/
public class ScriptSortParser implements SortParser {
private static final String STRING_SORT_TYPE = "string";
private static final String NUMBER_SORT_TYPE = "number";
@Override
public String[] names() {
return new String[]{"_script"};
@ -105,17 +112,9 @@ public class ScriptSortParser implements SortParser {
if (type == null) {
throw new SearchParseException(context, "_script sorting requires setting the type of the script");
}
SearchScript searchScript = context.scriptService().search(context.lookup(), scriptLang, script, scriptType, params);
IndexFieldData.XFieldComparatorSource fieldComparatorSource;
if ("string".equals(type)) {
fieldComparatorSource = StringScriptDataComparator.comparatorSource(searchScript);
} else if ("number".equals(type)) {
fieldComparatorSource = DoubleScriptDataComparator.comparatorSource(searchScript);
} else {
throw new SearchParseException(context, "custom script sort type [" + type + "] not supported");
}
final SearchScript searchScript = context.scriptService().search(context.lookup(), scriptLang, script, scriptType, params);
if ("string".equals(type) && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) {
if (STRING_SORT_TYPE.equals(type) && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) {
throw new SearchParseException(context, "type [string] doesn't support mode [" + sortMode + "]");
}
@ -125,6 +124,7 @@ public class ScriptSortParser implements SortParser {
// If nested_path is specified, then wrap the `fieldComparatorSource` in a `NestedFieldComparatorSource`
ObjectMapper objectMapper;
final Nested nested;
if (nestedPath != null) {
ObjectMappers objectMappers = context.mapperService().objectMapper(nestedPath);
if (objectMappers == null) {
@ -142,7 +142,58 @@ public class ScriptSortParser implements SortParser {
} else {
innerDocumentsFilter = context.filterCache().cache(objectMapper.nestedTypeFilter());
}
fieldComparatorSource = new NestedFieldComparatorSource(sortMode, fieldComparatorSource, rootDocumentsFilter, innerDocumentsFilter);
nested = new Nested(rootDocumentsFilter, innerDocumentsFilter);
} else {
nested = null;
}
final IndexFieldData.XFieldComparatorSource fieldComparatorSource;
switch (type) {
case STRING_SORT_TYPE:
fieldComparatorSource = new BytesRefFieldComparatorSource(null, null, sortMode, nested) {
@Override
protected SortedBinaryDocValues getValues(AtomicReaderContext context) {
searchScript.setNextReader(context);
final BinaryDocValues values = new BinaryDocValues() {
final BytesRef spare = new BytesRef();
@Override
public BytesRef get(int docID) {
searchScript.setNextDocId(docID);
spare.copyChars(searchScript.run().toString());
return spare;
}
};
return FieldData.singleton(values, null);
}
@Override
protected void setScorer(Scorer scorer) {
searchScript.setScorer(scorer);
}
};
break;
case NUMBER_SORT_TYPE:
// TODO: should we rather sort missing values last?
fieldComparatorSource = new DoubleValuesComparatorSource(null, Double.MAX_VALUE, sortMode, nested) {
@Override
protected SortedNumericDoubleValues getValues(AtomicReaderContext context) {
searchScript.setNextReader(context);
final NumericDoubleValues values = new NumericDoubleValues() {
@Override
public double get(int docID) {
searchScript.setNextDocId(docID);
return searchScript.runAsDouble();
}
};
return FieldData.singleton(values, null);
}
@Override
protected void setScorer(Scorer scorer) {
searchScript.setScorer(scorer);
}
};
break;
default:
throw new SearchParseException(context, "custom script sort type [" + type + "] not supported");
}
return new SortField("_script", fieldComparatorSource, reverse);

17
src/main/java/org/elasticsearch/search/sort/SortParseElement.java
View File

@ -28,14 +28,14 @@ import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.ObjectMappers;
import org.elasticsearch.index.mapper.core.NumberFieldMapper;
import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.query.ParsedFilter;
import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
import org.elasticsearch.index.search.nested.NonNestedDocsFilter;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.SearchParseException;
import org.elasticsearch.search.internal.SearchContext;
@ -190,7 +190,7 @@ public class SortParseElement implements SearchParseElement {
sortFields.add(SORT_DOC);
}
} else {
FieldMapper fieldMapper = context.smartNameFieldMapper(fieldName);
FieldMapper<?> fieldMapper = context.smartNameFieldMapper(fieldName);
if (fieldMapper == null) {
if (ignoreUnmapped) {
return;
@ -219,8 +219,7 @@ public class SortParseElement implements SearchParseElement {
sortMode = resolveDefaultSortMode(reverse);
}
IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.fieldData().getForField(fieldMapper)
.comparatorSource(missing, sortMode);
ObjectMapper objectMapper;
if (nestedPath != null) {
ObjectMappers objectMappers = context.mapperService().objectMapper(nestedPath);
@ -234,6 +233,7 @@ public class SortParseElement implements SearchParseElement {
} else {
objectMapper = context.mapperService().resolveClosestNestedObjectMapper(fieldName);
}
final Nested nested;
if (objectMapper != null && objectMapper.nested().isNested()) {
Filter rootDocumentsFilter = context.filterCache().cache(NonNestedDocsFilter.INSTANCE);
Filter innerDocumentsFilter;
@ -242,8 +242,13 @@ public class SortParseElement implements SearchParseElement {
} else {
innerDocumentsFilter = context.filterCache().cache(objectMapper.nestedTypeFilter());
}
fieldComparatorSource = new NestedFieldComparatorSource(sortMode, fieldComparatorSource, rootDocumentsFilter, innerDocumentsFilter);
nested = new Nested(rootDocumentsFilter, innerDocumentsFilter);
} else {
nested = null;
}
IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.fieldData().getForField(fieldMapper)
.comparatorSource(missing, sortMode, nested);
sortFields.add(new SortField(fieldMapper.names().indexName(), fieldComparatorSource, reverse));
}
}

3
src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java
View File

@ -28,17 +28,18 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.elasticsearch.test.ElasticsearchTestCase.UsesLuceneFieldCacheOnPurpose;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
/**
*
*/
@UsesLuceneFieldCacheOnPurpose
public class SimpleLuceneTests extends ElasticsearchTestCase {
@Test

18
src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTests.java
View File

@ -24,7 +24,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.search.*;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Strings;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
import org.elasticsearch.search.MultiValueMode;
import org.junit.Test;
@ -100,7 +100,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
TopFieldDocs topDocs;
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN))));
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null))));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one()));
@ -110,7 +110,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three()));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true)));
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true)));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
@ -167,14 +167,14 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
assertValues(bytesValues, 2, three());
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN))));
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null))));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs.length, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true)));
topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true)));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs.length, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
@ -226,7 +226,7 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN))));
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null))));
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
@ -242,12 +242,12 @@ public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests
assertThat(topDocs.scoreDocs[5].doc, equalTo(6));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("08"));
assertThat(topDocs.scoreDocs[6].doc, equalTo(1));
assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(BytesRefFieldComparatorSource.MAX_TERM));
assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(XFieldComparatorSource.MAX_TERM));
assertThat(topDocs.scoreDocs[7].doc, equalTo(5));
assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(BytesRefFieldComparatorSource.MAX_TERM));
assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(XFieldComparatorSource.MAX_TERM));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true)));
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true)));
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(6));

36
src/test/java/org/elasticsearch/index/fielddata/AbstractNumericFieldDataTests.java
View File

@ -91,14 +91,14 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
TopFieldDocs topDocs;
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN))));
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null))));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true)));
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true)));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
@ -145,42 +145,42 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
TopFieldDocs topDocs;
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN)))); // defaults to _last
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); // defaults to _last
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true))); // defaults to _last
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); // defaults to _last
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN))));
new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN, null))));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX), true)));
new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX, null), true)));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
assertThat(topDocs.scoreDocs[2].doc, equalTo(0));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MIN))));
new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MIN, null))));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MAX), true)));
new Sort(new SortField("value", indexFieldData.comparatorSource("1", MultiValueMode.MAX, null), true)));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
@ -323,7 +323,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN)))); // defaults to _last
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MIN, null)))); // defaults to _last
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
@ -344,7 +344,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
// assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX), true))); // defaults to _last
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.MAX, null), true))); // defaults to _last
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(6));
@ -366,7 +366,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
searcher = new IndexSearcher(DirectoryReader.open(writer, true));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM)))); // defaults to _last
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM, null)))); // defaults to _last
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
@ -388,7 +388,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
searcher = new IndexSearcher(DirectoryReader.open(writer, true));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM), true))); // defaults to _last
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.SUM, null), true))); // defaults to _last
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(6));
@ -410,7 +410,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
searcher = new IndexSearcher(DirectoryReader.open(writer, true));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG)))); // defaults to _last
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG, null)))); // defaults to _last
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
@ -432,7 +432,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
searcher = new IndexSearcher(DirectoryReader.open(writer, true));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG), true))); // defaults to _last
new Sort(new SortField("value", indexFieldData.comparatorSource(null, MultiValueMode.AVG, null), true))); // defaults to _last
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(6));
@ -453,7 +453,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
// assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN))));
new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MIN, null))));
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
@ -466,7 +466,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
assertThat(topDocs.scoreDocs[7].doc, equalTo(6));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX), true)));
new Sort(new SortField("value", indexFieldData.comparatorSource("_first", MultiValueMode.MAX, null), true)));
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
@ -479,7 +479,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
assertThat(topDocs.scoreDocs[7].doc, equalTo(7));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource("-9", MultiValueMode.MIN))));
new Sort(new SortField("value", indexFieldData.comparatorSource("-9", MultiValueMode.MIN, null))));
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
@ -492,7 +492,7 @@ public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImp
assertThat(topDocs.scoreDocs[7].doc, equalTo(6));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource("9", MultiValueMode.MAX), true)));
new Sort(new SortField("value", indexFieldData.comparatorSource("9", MultiValueMode.MAX, null), true)));
assertThat(topDocs.totalHits, equalTo(8));
assertThat(topDocs.scoreDocs.length, equalTo(8));
assertThat(topDocs.scoreDocs[0].doc, equalTo(6));

50
src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTests.java
View File

@ -35,9 +35,9 @@ import org.elasticsearch.common.lucene.search.NotFilter;
import org.elasticsearch.common.lucene.search.XFilteredQuery;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData;
import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
import org.elasticsearch.search.MultiValueMode;
import org.junit.Test;
@ -237,7 +237,7 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
final IndexFieldData indexFieldData = getForField("value");
final String missingValue = values[1];
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
XFieldComparatorSource comparator = indexFieldData.comparatorSource(missingValue, MultiValueMode.MIN);
XFieldComparatorSource comparator = indexFieldData.comparatorSource(missingValue, MultiValueMode.MIN, null);
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse)));
assertEquals(numDocs, topDocs.totalHits);
BytesRef previousValue = reverse ? UnicodeUtil.BIG_TERM : new BytesRef();
@ -293,7 +293,7 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
}
final IndexFieldData indexFieldData = getForField("value");
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
XFieldComparatorSource comparator = indexFieldData.comparatorSource(first ? "_first" : "_last", MultiValueMode.MIN);
XFieldComparatorSource comparator = indexFieldData.comparatorSource(first ? "_first" : "_last", MultiValueMode.MIN, null);
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse)));
assertEquals(numDocs, topDocs.totalHits);
BytesRef previousValue = first ? null : reverse ? UnicodeUtil.BIG_TERM : new BytesRef();
@ -360,13 +360,13 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
}
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
IndexFieldData<?> fieldData = getForField("text");
final BytesRef missingValue;
final Object missingValue;
switch (randomInt(4)) {
case 0:
missingValue = new BytesRef();
missingValue = "_first";
break;
case 1:
missingValue = BytesRefFieldComparatorSource.MAX_TERM;
missingValue = "_last";
break;
case 2:
missingValue = new BytesRef(RandomPicks.randomFrom(getRandom(), values));
@ -375,10 +375,10 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
missingValue = new BytesRef(TestUtil.randomSimpleString(getRandom()));
break;
}
BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode);
Filter parentFilter = new TermFilter(new Term("type", "parent"));
Filter childFilter = new NotFilter(parentFilter);
NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
Nested nested = new Nested(parentFilter, childFilter);
BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode, nested);
ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
Sort sort = new Sort(new SortField("text", nestedComparatorSource));
TopFieldDocs topDocs = searcher.search(query, randomIntBetween(1, numParents), sort);
@ -389,26 +389,36 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImpl
assertTrue("expected " + docID + " to be a parent", parents.get(docID));
BytesRef cmpValue = null;
for (int child = parents.prevSetBit(docID - 1) + 1; child < docID; ++child) {
String[] vals = searcher.doc(child).getValues("text");
if (vals.length == 0) {
vals = new String[] {missingValue.utf8ToString()};
String[] sVals = searcher.doc(child).getValues("text");
final BytesRef[] vals;
if (sVals.length == 0) {
vals = new BytesRef[0];
} else {
vals = new BytesRef[sVals.length];
for (int j = 0; j < vals.length; ++j) {
vals[j] = new BytesRef(sVals[j]);
}
}
for (String value : vals) {
final BytesRef bytesValue = new BytesRef(value);
for (BytesRef value : vals) {
if (cmpValue == null) {
cmpValue = bytesValue;
} else if (sortMode == MultiValueMode.MIN && bytesValue.compareTo(cmpValue) < 0) {
cmpValue = bytesValue;
} else if (sortMode == MultiValueMode.MAX && bytesValue.compareTo(cmpValue) > 0) {
cmpValue = bytesValue;
cmpValue = value;
} else if (sortMode == MultiValueMode.MIN && value.compareTo(cmpValue) < 0) {
cmpValue = value;
} else if (sortMode == MultiValueMode.MAX && value.compareTo(cmpValue) > 0) {
cmpValue = value;
}
}
}
if (cmpValue == null) {
cmpValue = missingValue;
if ("_first".equals(missingValue)) {
cmpValue = new BytesRef();
} else if ("_last".equals(missingValue)) {
cmpValue = XFieldComparatorSource.MAX_TERM;
} else {
cmpValue = (BytesRef) missingValue;
}
}
if (previous != null) {
assertNotNull(cmpValue);
assertTrue(previous.utf8ToString() + " / " + cmpValue.utf8ToString(), previous.compareTo(cmpValue) <= 0);
}
previous = cmpValue;

5
src/test/java/org/elasticsearch/index/fielddata/NoOrdinalsStringFieldDataTests.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.mapper.FieldMapper.Names;
import org.elasticsearch.search.MultiValueMode;
@ -60,8 +61,8 @@ public class NoOrdinalsStringFieldDataTests extends PagedBytesStringFieldDataTes
}
@Override
public XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode) {
return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
public XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
}
@Override

5
src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java
View File

@ -27,6 +27,7 @@ import org.apache.lucene.search.*;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
@ -140,7 +141,7 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTests {
public void testSorting() throws Exception {
IndexFieldData indexFieldData = getForField(childType);
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
IndexFieldData.XFieldComparatorSource comparator = indexFieldData.comparatorSource("_last", MultiValueMode.MIN);
IndexFieldData.XFieldComparatorSource comparator = indexFieldData.comparatorSource("_last", MultiValueMode.MIN, null);
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField(ParentFieldMapper.NAME, comparator, false)));
assertThat(topDocs.totalHits, equalTo(8));
@ -160,7 +161,7 @@ public class ParentChildFieldDataTests extends AbstractFieldDataTests {
assertThat(topDocs.scoreDocs[6].doc, equalTo(6));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("2"));
assertThat(topDocs.scoreDocs[7].doc, equalTo(7));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]), equalTo(IndexFieldData.XFieldComparatorSource.MAX_TERM));
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]), equalTo(XFieldComparatorSource.MAX_TERM));
topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField(ParentFieldMapper.NAME, comparator, true)));
assertThat(topDocs.totalHits, equalTo(8));

2
src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java
View File

@ -108,7 +108,7 @@ public class MultiOrdinalsTests extends ElasticsearchTestCase {
});
Ordinals ords = creationMultiOrdinals(builder);
RandomAccessOrds docs = ords.ordinals();
final SortedDocValues singleOrds = MultiValueMode.MIN.select(docs, -1);
final SortedDocValues singleOrds = MultiValueMode.MIN.select(docs);
int docId = ordsAndIds.get(0).id;
List<Long> docOrds = new ArrayList<>();
for (OrdAndId ordAndId : ordsAndIds) {

31
src/test/java/org/elasticsearch/index/search/nested/AbstractNumberNestedSortingTests.java
View File

@ -33,6 +33,8 @@ import org.elasticsearch.common.lucene.search.NotFilter;
import org.elasticsearch.common.lucene.search.XFilteredQuery;
import org.elasticsearch.index.fielddata.AbstractFieldDataTests;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.search.MultiValueMode;
import org.junit.Test;
@ -206,10 +208,9 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData
MultiValueMode sortMode = MultiValueMode.SUM;
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false));
IndexFieldData.XFieldComparatorSource innerFieldComparator = createInnerFieldComparator("field2", sortMode, null);
Filter parentFilter = new TermFilter(new Term("__type", "parent"));
Filter childFilter = new NotFilter(parentFilter);
NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, null, new Nested(parentFilter, childFilter));
ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
@ -243,7 +244,7 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData
assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(9));
childFilter = new TermFilter(new Term("filter_1", "T"));
nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
nestedComparatorSource = createFieldComparator("field2", sortMode, null, new Nested(parentFilter, childFilter));
query = new ToParentBlockJoinQuery(
new XFilteredQuery(new MatchAllDocsQuery(), childFilter),
new FixedBitSetCachingWrapperFilter(parentFilter),
@ -279,8 +280,7 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData
assertThat(topDocs.scoreDocs[4].doc, equalTo(3));
assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(9));
innerFieldComparator = createInnerFieldComparator("field2", sortMode, 127);
nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
nestedComparatorSource = createFieldComparator("field2", sortMode, 127, new Nested(parentFilter, childFilter));
sort = new Sort(new SortField("field2", nestedComparatorSource, true));
topDocs = searcher.search(new TermQuery(new Term("__type", "parent")), 5, sort);
assertThat(topDocs.totalHits, equalTo(8));
@ -296,8 +296,7 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData
assertThat(topDocs.scoreDocs[4].doc, equalTo(7));
assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(8));
innerFieldComparator = createInnerFieldComparator("field2", sortMode, -127);
nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter));
sort = new Sort(new SortField("field2", nestedComparatorSource));
topDocs = searcher.search(new TermQuery(new Term("__type", "parent")), 5, sort);
assertThat(topDocs.totalHits, equalTo(8));
@ -315,33 +314,33 @@ public abstract class AbstractNumberNestedSortingTests extends AbstractFieldData
// Moved to method, because floating point based XFieldComparatorSource have different outcome for SortMode avg,
// than integral number based implementations...
assertAvgScoreMode(parentFilter, searcher, innerFieldComparator);
assertAvgScoreMode(parentFilter, searcher);
searcher.getIndexReader().close();
}
protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher, IndexFieldData.XFieldComparatorSource innerFieldComparator) throws IOException {
protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher) throws IOException {
MultiValueMode sortMode = MultiValueMode.AVG;
Filter childFilter = new NotFilter(parentFilter);
NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter));
Query query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
TopDocs topDocs = searcher.search(query, 5, sort);
assertThat(topDocs.totalHits, equalTo(7));
assertThat(topDocs.scoreDocs.length, equalTo(5));
assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
assertThat(topDocs.scoreDocs[0].doc, equalTo(11));
assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).intValue(), equalTo(2));
assertThat(topDocs.scoreDocs[1].doc, equalTo(11));
assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(2));
assertThat(topDocs.scoreDocs[2].doc, equalTo(3));
assertThat(topDocs.scoreDocs[1].doc, equalTo(3));
assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).intValue(), equalTo(3));
assertThat(topDocs.scoreDocs[2].doc, equalTo(7));
assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).intValue(), equalTo(3));
assertThat(topDocs.scoreDocs[3].doc, equalTo(15));
assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).intValue(), equalTo(3));
assertThat(topDocs.scoreDocs[4].doc, equalTo(19));
assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(3));
assertThat(((Number) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).intValue(), equalTo(4));
}
protected abstract IndexableField createField(String name, int value, Field.Store store);
protected abstract IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue);
protected abstract IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested);
}

12
src/test/java/org/elasticsearch/index/search/nested/DoubleNestedSortingTests.java
View File

@ -29,9 +29,11 @@ import org.elasticsearch.common.lucene.search.NotFilter;
import org.elasticsearch.common.lucene.search.XFilteredQuery;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
@ -47,9 +49,9 @@ public class DoubleNestedSortingTests extends AbstractNumberNestedSortingTests {
}
@Override
protected IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue) {
protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) {
DoubleArrayIndexFieldData fieldData = getForField(fieldName);
return new DoubleValuesComparatorSource(fieldData, missingValue, sortMode);
return new DoubleValuesComparatorSource(fieldData, missingValue, sortMode, nested);
}
@Override
@ -57,10 +59,10 @@ public class DoubleNestedSortingTests extends AbstractNumberNestedSortingTests {
return new DoubleField(name, value, store);
}
protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher, IndexFieldData.XFieldComparatorSource innerFieldComparator) throws IOException {
protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher) throws IOException {
MultiValueMode sortMode = MultiValueMode.AVG;
Filter childFilter = new NotFilter(parentFilter);
NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter));
Query query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
TopDocs topDocs = searcher.search(query, 5, sort);

12
src/test/java/org/elasticsearch/index/search/nested/FloatNestedSortingTests.java
View File

@ -29,9 +29,11 @@ import org.elasticsearch.common.lucene.search.NotFilter;
import org.elasticsearch.common.lucene.search.XFilteredQuery;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.fielddata.plain.FloatArrayIndexFieldData;
import org.elasticsearch.search.MultiValueMode;
import java.io.IOException;
@ -39,7 +41,7 @@ import static org.hamcrest.Matchers.equalTo;
/**
*/
public class FloatNestedSortingTests extends AbstractNumberNestedSortingTests {
public class FloatNestedSortingTests extends DoubleNestedSortingTests {
@Override
protected FieldDataType getFieldDataType() {
@ -47,9 +49,9 @@ public class FloatNestedSortingTests extends AbstractNumberNestedSortingTests {
}
@Override
protected IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue) {
protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) {
FloatArrayIndexFieldData fieldData = getForField(fieldName);
return new FloatValuesComparatorSource(fieldData, missingValue, sortMode);
return new FloatValuesComparatorSource(fieldData, missingValue, sortMode, nested);
}
@Override
@ -60,7 +62,7 @@ public class FloatNestedSortingTests extends AbstractNumberNestedSortingTests {
protected void assertAvgScoreMode(Filter parentFilter, IndexSearcher searcher, IndexFieldData.XFieldComparatorSource innerFieldComparator) throws IOException {
MultiValueMode sortMode = MultiValueMode.AVG;
Filter childFilter = new NotFilter(parentFilter);
NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerFieldComparator, parentFilter, childFilter);
XFieldComparatorSource nestedComparatorSource = createFieldComparator("field2", sortMode, -127, new Nested(parentFilter, childFilter));
Query query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
TopDocs topDocs = searcher.search(query, 5, sort);

7
src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java
View File

@ -23,9 +23,10 @@ import org.apache.lucene.document.LongField;
import org.apache.lucene.index.IndexableField;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
import org.elasticsearch.search.MultiValueMode;
import org.elasticsearch.index.fielddata.plain.PackedArrayIndexFieldData;
import org.elasticsearch.search.MultiValueMode;
/**
*/
@ -37,9 +38,9 @@ public class LongNestedSortingTests extends AbstractNumberNestedSortingTests {
}
@Override
protected IndexFieldData.XFieldComparatorSource createInnerFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue) {
protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, Object missingValue, Nested nested) {
PackedArrayIndexFieldData fieldData = getForField(fieldName);
return new LongValuesComparatorSource(fieldData, missingValue, sortMode);
return new LongValuesComparatorSource(fieldData, missingValue, sortMode, nested);
}
@Override

11
src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
View File

@ -37,6 +37,7 @@ import org.elasticsearch.common.lucene.search.XFilteredQuery;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
import org.elasticsearch.search.MultiValueMode;
@ -106,8 +107,7 @@ public class NestedSortingTests extends AbstractFieldDataTests {
private TopDocs getTopDocs(IndexSearcher searcher, IndexFieldData<?> indexFieldData, String missingValue, MultiValueMode sortMode, int n, boolean reverse) throws IOException {
Filter parentFilter = new TermFilter(new Term("__type", "parent"));
Filter childFilter = new TermFilter(new Term("__type", "child"));
XFieldComparatorSource innerSource = indexFieldData.comparatorSource(missingValue, sortMode);
NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
XFieldComparatorSource nestedComparatorSource = indexFieldData.comparatorSource(missingValue, sortMode, new Nested(parentFilter, childFilter));
Query query = new ConstantScoreQuery(parentFilter);
Sort sort = new Sort(new SortField("f", nestedComparatorSource, reverse));
return searcher.search(query, n, sort);
@ -271,10 +271,9 @@ public class NestedSortingTests extends AbstractFieldDataTests {
MultiValueMode sortMode = MultiValueMode.MIN;
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false));
PagedBytesIndexFieldData indexFieldData = getForField("field2");
BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode);
Filter parentFilter = new TermFilter(new Term("__type", "parent"));
Filter childFilter = new NotFilter(parentFilter);
NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, new Nested(parentFilter, childFilter));
ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new FixedBitSetCachingWrapperFilter(parentFilter), ScoreMode.None);
Sort sort = new Sort(new SortField("field2", nestedComparatorSource));
@ -293,7 +292,7 @@ public class NestedSortingTests extends AbstractFieldDataTests {
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("i"));
sortMode = MultiValueMode.MAX;
nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, new Nested(parentFilter, childFilter));
sort = new Sort(new SortField("field2", nestedComparatorSource, true));
topDocs = searcher.search(query, 5, sort);
assertThat(topDocs.totalHits, equalTo(7));
@ -311,7 +310,7 @@ public class NestedSortingTests extends AbstractFieldDataTests {
childFilter = new AndFilter(Arrays.asList(new NotFilter(parentFilter), new TermFilter(new Term("filter_1", "T"))));
nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
nestedComparatorSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode, new Nested(parentFilter, childFilter));
query = new ToParentBlockJoinQuery(
new XFilteredQuery(new MatchAllDocsQuery(), childFilter),
new FixedBitSetCachingWrapperFilter(parentFilter),

12
src/test/java/org/elasticsearch/nested/SimpleNestedTests.java
View File

@ -1263,11 +1263,11 @@ public class SimpleNestedTests extends ElasticsearchIntegrationTest {
assertHitCount(searchResponse, 3);
assertThat(searchResponse.getHits().getHits().length, equalTo(3));
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("3"));
assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("0"));
assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2"));
assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("2"));
assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("3"));
searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
@ -1282,11 +1282,11 @@ public class SimpleNestedTests extends ElasticsearchIntegrationTest {
assertHitCount(searchResponse, 3);
assertThat(searchResponse.getHits().getHits().length, equalTo(3));
assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("2"));
assertThat(searchResponse.getHits().getHits()[0].sortValues()[0].toString(), equalTo("3"));
assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("2"));
assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("1"));
assertThat(searchResponse.getHits().getHits()[1].sortValues()[0].toString(), equalTo("2"));
assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3"));
assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("0"));
assertThat(searchResponse.getHits().getHits()[2].sortValues()[0].toString(), equalTo("1"));
// Sort mode: avg with filter
searchResponse = client().prepareSearch()

545
src/test/java/org/elasticsearch/search/MultiValueModeTests.java Normal file
View File

@ -0,0 +1,545 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.apache.lucene.index.*;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.index.fielddata.FieldData;
import org.elasticsearch.index.fielddata.NumericDoubleValues;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.test.ElasticsearchTestCase;
import java.util.Arrays;
public class MultiValueModeTests extends ElasticsearchTestCase {
private static FixedBitSet randomRootDocs(int maxDoc) {
FixedBitSet set = new FixedBitSet(maxDoc);
for (int i = 0; i < maxDoc; ++i) {
if (randomBoolean()) {
set.set(i);
}
}
// the last doc must be a root doc
set.set(maxDoc - 1);
return set;
}
private static FixedBitSet randomInnerDocs(FixedBitSet rootDocs) {
FixedBitSet innerDocs = new FixedBitSet(rootDocs.length());
for (int i = 0; i < innerDocs.length(); ++i) {
if (!rootDocs.get(i) && randomBoolean()) {
innerDocs.set(i);
}
}
return innerDocs;
}
public void testSingleValuedLongs() {
final int numDocs = scaledRandomIntBetween(1, 100);
final long[] array = new long[numDocs];
final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs);
for (int i = 0; i < array.length; ++i) {
if (randomBoolean()) {
array[i] = randomLong();
if (docsWithValue != null) {
docsWithValue.set(i);
}
} else if (docsWithValue != null && randomBoolean()) {
docsWithValue.set(i);
}
}
final NumericDocValues singleValues = new NumericDocValues() {
@Override
public long get(int docID) {
return array[docID];
}
};
final SortedNumericDocValues multiValues = DocValues.singleton(singleValues, docsWithValue);
verify(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verify(multiValues, numDocs, rootDocs, innerDocs);
}
public void testMultiValuedLongs() {
final int numDocs = scaledRandomIntBetween(1, 100);
final long[][] array = new long[numDocs][];
for (int i = 0; i < numDocs; ++i) {
final long[] values = new long[randomInt(4)];
for (int j = 0; j < values.length; ++j) {
values[j] = randomLong();
}
Arrays.sort(values);
array[i] = values;
}
final SortedNumericDocValues multiValues = new SortedNumericDocValues() {
int doc;
@Override
public long valueAt(int index) {
return array[doc][index];
}
@Override
public void setDocument(int doc) {
this.doc = doc;
}
@Override
public int count() {
return array[doc].length;
}
};
verify(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verify(multiValues, numDocs, rootDocs, innerDocs);
}
private void verify(SortedNumericDocValues values, int maxDoc) {
for (long missingValue : new long[] { 0, randomLong() }) {
for (MultiValueMode mode : MultiValueMode.values()) {
final NumericDocValues selected = mode.select(values, missingValue);
for (int i = 0; i < maxDoc; ++i) {
final long actual = selected.get(i);
long expected;
values.setDocument(i);
int numValues = values.count();
if (numValues == 0) {
expected = missingValue;
} else {
expected = mode.startLong();
for (int j = 0; j < numValues; ++j) {
expected = mode.apply(expected, values.valueAt(j));
}
expected = mode.reduce(expected, numValues);
}
assertEquals(mode.toString() + " docId=" + i, expected, actual);
}
}
}
}
private void verify(SortedNumericDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) {
for (long missingValue : new long[] { 0, randomLong() }) {
for (MultiValueMode mode : MultiValueMode.values()) {
final NumericDocValues selected = mode.select(values, missingValue, rootDocs, innerDocs, maxDoc);
int prevRoot = -1;
for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
final long actual = selected.get(root);
long expected = mode.startLong();
int numValues = 0;
for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
values.setDocument(child);
for (int j = 0; j < values.count(); ++j) {
expected = mode.apply(expected, values.valueAt(j));
++numValues;
}
}
if (numValues == 0) {
expected = missingValue;
} else {
expected = mode.reduce(expected, numValues);
}
assertEquals(mode.toString() + " docId=" + root, expected, actual);
prevRoot = root;
}
}
}
}
public void testSingleValuedDoubles() {
final int numDocs = scaledRandomIntBetween(1, 100);
final double[] array = new double[numDocs];
final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs);
for (int i = 0; i < array.length; ++i) {
if (randomBoolean()) {
array[i] = randomDouble();
if (docsWithValue != null) {
docsWithValue.set(i);
}
} else if (docsWithValue != null && randomBoolean()) {
docsWithValue.set(i);
}
}
final NumericDoubleValues singleValues = new NumericDoubleValues() {
@Override
public double get(int docID) {
return array[docID];
}
};
final SortedNumericDoubleValues multiValues = FieldData.singleton(singleValues, docsWithValue);
verify(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verify(multiValues, numDocs, rootDocs, innerDocs);
}
public void testMultiValuedDoubles() {
final int numDocs = scaledRandomIntBetween(1, 100);
final double[][] array = new double[numDocs][];
for (int i = 0; i < numDocs; ++i) {
final double[] values = new double[randomInt(4)];
for (int j = 0; j < values.length; ++j) {
values[j] = randomDouble();
}
Arrays.sort(values);
array[i] = values;
}
final SortedNumericDoubleValues multiValues = new SortedNumericDoubleValues() {
int doc;
@Override
public double valueAt(int index) {
return array[doc][index];
}
@Override
public void setDocument(int doc) {
this.doc = doc;
}
@Override
public int count() {
return array[doc].length;
}
};
verify(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verify(multiValues, numDocs, rootDocs, innerDocs);
}
private void verify(SortedNumericDoubleValues values, int maxDoc) {
for (long missingValue : new long[] { 0, randomLong() }) {
for (MultiValueMode mode : MultiValueMode.values()) {
final NumericDoubleValues selected = mode.select(values, missingValue);
for (int i = 0; i < maxDoc; ++i) {
final double actual = selected.get(i);
double expected;
values.setDocument(i);
int numValues = values.count();
if (numValues == 0) {
expected = missingValue;
} else {
expected = mode.startLong();
for (int j = 0; j < numValues; ++j) {
expected = mode.apply(expected, values.valueAt(j));
}
expected = mode.reduce(expected, numValues);
}
assertEquals(mode.toString() + " docId=" + i, expected, actual, 0.1);
}
}
}
}
private void verify(SortedNumericDoubleValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) {
for (long missingValue : new long[] { 0, randomLong() }) {
for (MultiValueMode mode : MultiValueMode.values()) {
final NumericDoubleValues selected = mode.select(values, missingValue, rootDocs, innerDocs, maxDoc);
int prevRoot = -1;
for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
final double actual = selected.get(root);
double expected = mode.startLong();
int numValues = 0;
for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
values.setDocument(child);
for (int j = 0; j < values.count(); ++j) {
expected = mode.apply(expected, values.valueAt(j));
++numValues;
}
}
if (numValues == 0) {
expected = missingValue;
} else {
expected = mode.reduce(expected, numValues);
}
assertEquals(mode.toString() + " docId=" + root, expected, actual, 0.1);
prevRoot = root;
}
}
}
}
public void testSingleValuedStrings() {
final int numDocs = scaledRandomIntBetween(1, 100);
final BytesRef[] array = new BytesRef[numDocs];
final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs);
for (int i = 0; i < array.length; ++i) {
if (randomBoolean()) {
array[i] = new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8));
if (docsWithValue != null) {
docsWithValue.set(i);
}
} else {
array[i] = new BytesRef();
if (docsWithValue != null && randomBoolean()) {
docsWithValue.set(i);
}
}
}
final BinaryDocValues singleValues = new BinaryDocValues() {
@Override
public BytesRef get(int docID) {
return BytesRef.deepCopyOf(array[docID]);
}
};
final SortedBinaryDocValues multiValues = FieldData.singleton(singleValues, docsWithValue);
verify(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verify(multiValues, numDocs, rootDocs, innerDocs);
}
public void testMultiValuedStrings() {
final int numDocs = scaledRandomIntBetween(1, 100);
final BytesRef[][] array = new BytesRef[numDocs][];
for (int i = 0; i < numDocs; ++i) {
final BytesRef[] values = new BytesRef[randomInt(4)];
for (int j = 0; j < values.length; ++j) {
values[j] = new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8));
}
Arrays.sort(values);
array[i] = values;
}
final SortedBinaryDocValues multiValues = new SortedBinaryDocValues() {
int doc;
@Override
public BytesRef valueAt(int index) {
return BytesRef.deepCopyOf(array[doc][index]);
}
@Override
public void setDocument(int doc) {
this.doc = doc;
}
@Override
public int count() {
return array[doc].length;
}
};
verify(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verify(multiValues, numDocs, rootDocs, innerDocs);
}
private void verify(SortedBinaryDocValues values, int maxDoc) {
for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8)) }) {
for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) {
final BinaryDocValues selected = mode.select(values, missingValue);
for (int i = 0; i < maxDoc; ++i) {
final BytesRef actual = selected.get(i);
BytesRef expected = null;
values.setDocument(i);
int numValues = values.count();
if (numValues == 0) {
expected = missingValue;
} else {
for (int j = 0; j < numValues; ++j) {
if (expected == null) {
expected = BytesRef.deepCopyOf(values.valueAt(j));
} else {
expected = mode.apply(expected, BytesRef.deepCopyOf(values.valueAt(j)));
}
}
if (expected == null) {
expected = missingValue;
}
}
assertEquals(mode.toString() + " docId=" + i, expected, actual);
}
}
}
}
private void verify(SortedBinaryDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) {
for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8)) }) {
for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) {
final BinaryDocValues selected = mode.select(values, missingValue, rootDocs, innerDocs, maxDoc);
int prevRoot = -1;
for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
final BytesRef actual = selected.get(root);
BytesRef expected = null;
for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
values.setDocument(child);
for (int j = 0; j < values.count(); ++j) {
if (expected == null) {
expected = BytesRef.deepCopyOf(values.valueAt(j));
} else {
expected = mode.apply(expected, values.valueAt(j));
}
expected = mode.apply(expected, BytesRef.deepCopyOf(values.valueAt(j)));
}
}
if (expected == null) {
expected = missingValue;
}
assertEquals(mode.toString() + " docId=" + root, expected, actual);
prevRoot = root;
}
}
}
}
public void testSingleValuedOrds() {
final int numDocs = scaledRandomIntBetween(1, 100);
final int[] array = new int[numDocs];
for (int i = 0; i < array.length; ++i) {
if (randomBoolean()) {
array[i] = randomInt(1000);
} else {
array[i] = -1;
}
}
final SortedDocValues singleValues = new SortedDocValues() {
@Override
public int getOrd(int docID) {
return array[docID];
}
@Override
public BytesRef lookupOrd(int ord) {
throw new UnsupportedOperationException();
}
@Override
public int getValueCount() {
return 1 << 20;
}
};
final RandomAccessOrds multiValues = (RandomAccessOrds) DocValues.singleton(singleValues);
verify(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verify(multiValues, numDocs, rootDocs, innerDocs);
}
public void testMultiValuedOrds() {
final int numDocs = scaledRandomIntBetween(1, 100);
final long[][] array = new long[numDocs][];
for (int i = 0; i < numDocs; ++i) {
final long[] values = new long[randomInt(4)];
for (int j = 0; j < values.length; ++j) {
values[j] = j == 0 ? randomInt(1000) : values[j - 1] + 1 + randomInt(1000);
}
array[i] = values;
}
final RandomAccessOrds multiValues = new RandomAccessOrds() {
int doc;
@Override
public long ordAt(int index) {
return array[doc][index];
}
@Override
public int cardinality() {
return array[doc].length;
}
@Override
public long nextOrd() {
throw new UnsupportedOperationException();
}
@Override
public void setDocument(int docID) {
this.doc = docID;
}
@Override
public BytesRef lookupOrd(long ord) {
throw new UnsupportedOperationException();
}
@Override
public long getValueCount() {
return 1 << 20;
}
};
verify(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);
final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
verify(multiValues, numDocs, rootDocs, innerDocs);
}
private void verify(RandomAccessOrds values, int maxDoc) {
for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) {
final SortedDocValues selected = mode.select(values);
for (int i = 0; i < maxDoc; ++i) {
final long actual = selected.getOrd(i);
int expected = -1;
values.setDocument(i);
for (int j = 0; j < values.cardinality(); ++j) {
if (expected == -1) {
expected = (int) values.ordAt(j);
} else {
expected = mode.applyOrd(expected, (int) values.ordAt(j));
}
}
assertEquals(mode.toString() + " docId=" + i, expected, actual);
}
}
}
private void verify(RandomAccessOrds values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) {
for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) {
final SortedDocValues selected = mode.select(values, rootDocs, innerDocs);
int prevRoot = -1;
for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) {
final int actual = selected.getOrd(root);
int expected = -1;
for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) {
values.setDocument(child);
for (int j = 0; j < values.cardinality(); ++j) {
if (expected == -1) {
expected = (int) values.ordAt(j);
} else {
expected = mode.applyOrd(expected, (int) values.ordAt(j));
}
}
}
assertEquals(mode.toString() + " docId=" + root, expected, actual);
prevRoot = root;
}
}
}
}

30
src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java
View File

@ -23,6 +23,7 @@ import com.carrotsearch.randomizedtesting.annotations.*;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.AbstractRandomizedTest;
import org.apache.lucene.util.LuceneTestCase;
@ -39,10 +40,7 @@ import org.elasticsearch.test.cache.recycler.MockBigArrays;
import org.elasticsearch.test.cache.recycler.MockPageCacheRecycler;
import org.elasticsearch.test.junit.listeners.LoggingListener;
import org.elasticsearch.test.store.MockDirectoryHelper;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.*;
import java.io.Closeable;
import java.io.File;
@ -99,6 +97,21 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest {
}
@Before
public void cleanFieldCache() {
FieldCache.DEFAULT.purgeAllCaches();
}
@After
public void ensureNoFieldCacheUse() {
// We use the lucene comparators, and by default they work on field cache.
// However, given the way that we use them, field cache should NEVER get loaded.
if (getClass().getAnnotation(UsesLuceneFieldCacheOnPurpose.class) == null) {
FieldCache.CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries();
assertEquals("fieldcache must never be used, got=" + Arrays.toString(entries), 0, entries.length);
}
}
/**
* Runs the code block for 10 seconds waiting for no assertion to trip.
*/
@ -395,6 +408,15 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest {
int version();
}
/**
* Most tests don't use {@link FieldCache} but some of them might do.
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.TYPE})
@Ignore
public @interface UsesLuceneFieldCacheOnPurpose {
}
/**
* Returns a global compatibility version that is set via the
* {@value #TESTS_COMPATIBILITY} or {@value #TESTS_BACKWARDS_COMPATIBILITY_VERSION} system property.