mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-17 10:25:15 +00:00
Configurable sort order for missing string values.
This commit allows for configuring the sort order of missing values in BytesRef comparators (used for strings) with the following options: - _first: missing values will appear in the first positions, - _last: missing values will appear in the last positions (default), - <any value>: documents with missing sort value will use the given value when sorting. Since the default is _last, sorting by string value will have a different behavior than in previous versions of elasticsearch which used to insert missing value in the first positions when sorting in ascending order. Implementation notes: - Nested sorting is supported through the implementation of NestedWrappableComparator, - BytesRefValComparator was mostly broken since no field data implementation was using it, it is now tested through NoOrdinalsStringFieldDataTests, - Specialized BytesRefOrdValComparators have been removed now that the ordinals rely on packed arrays instead of raw arrays, - Field data tests hierarchy has been changed so that the numeric tests don't inherit from the string tests anymore, - When _first or _last is used, internally the comparators are told to use null or BytesRefFieldComparatorSource.MAX_TERM to replace missing values (depending on the sort order), - BytesRefValComparator just replaces missing values with the provided value and uses them for comparisons, - BytesRefOrdValComparator multiplies ordinals by 4 so that it can find ordinals for the missing value and the bottom value which are directly comparable with the segment ordinals. For example, if the segment values and ordinals are (a,1) and (b,2), they will be stored internally as (a,4) and (b,8) and if the missing value is 'ab', it will be assigned 6 as an ordinal, since it is between 'a' and 'b'. Then if the bottom value is 'abc', it will be assigned 7 as an ordinal since if it between 'ab' and 'b'. Closes #896
This commit is contained in:
Adrien Grand
parent
2c939847b4
commit
db46946d16
@ -21,6 +21,9 @@ package org.elasticsearch.index.fielddata.fieldcomparator;
|
||||
|
||||
import org.apache.lucene.search.FieldComparator;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.UnicodeUtil;
|
||||
import org.elasticsearch.ElasticSearchIllegalArgumentException;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
|
||||
import java.io.IOException;
|
||||
@ -29,12 +32,23 @@ import java.io.IOException;
|
||||
*/
|
||||
public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparatorSource {
|
||||
|
||||
/** UTF-8 term containing a single code point: {@link Character#MAX_CODE_POINT} which will compare greater than all other index terms
|
||||
* since {@link Character#MAX_CODE_POINT} is a noncharacter and thus shouldn't appear in an index term. */
|
||||
public static final BytesRef MAX_TERM;
|
||||
static {
|
||||
MAX_TERM = new BytesRef();
|
||||
final char[] chars = Character.toChars(Character.MAX_CODE_POINT);
|
||||
UnicodeUtil.UTF16toUTF8(chars, 0, chars.length, MAX_TERM);
|
||||
}
|
||||
|
||||
private final IndexFieldData<?> indexFieldData;
|
||||
private final SortMode sortMode;
|
||||
private final Object missingValue;
|
||||
|
||||
public BytesRefFieldComparatorSource(IndexFieldData<?> indexFieldData, SortMode sortMode) {
|
||||
public BytesRefFieldComparatorSource(IndexFieldData<?> indexFieldData, Object missingValue, SortMode sortMode) {
|
||||
this.indexFieldData = indexFieldData;
|
||||
this.sortMode = sortMode;
|
||||
this.missingValue = missingValue;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -45,9 +59,25 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat
|
||||
@Override
|
||||
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
|
||||
assert fieldname.equals(indexFieldData.getFieldNames().indexName());
|
||||
if (indexFieldData.valuesOrdered() && indexFieldData instanceof IndexFieldData.WithOrdinals) {
|
||||
return new BytesRefOrdValComparator((IndexFieldData.WithOrdinals<?>) indexFieldData, numHits, sortMode);
|
||||
BytesRef missingBytes = null;
|
||||
if (missingValue == null || "_last".equals(missingValue)) {
|
||||
missingBytes = reversed ? null : MAX_TERM;
|
||||
} else if ("_first".equals(missingValue)) {
|
||||
missingBytes = reversed ? MAX_TERM : null;
|
||||
} else if (missingValue instanceof BytesRef) {
|
||||
missingBytes = (BytesRef) missingValue;
|
||||
} else if (missingValue instanceof String) {
|
||||
missingBytes = new BytesRef((String) missingValue);
|
||||
} else if (missingValue instanceof byte[]) {
|
||||
missingBytes = new BytesRef((byte[]) missingValue);
|
||||
} else {
|
||||
throw new ElasticSearchIllegalArgumentException("Unsupported missing value: " + missingValue);
|
||||
}
|
||||
return new BytesRefValComparator(indexFieldData, numHits, sortMode);
|
||||
|
||||
if (indexFieldData.valuesOrdered() && indexFieldData instanceof IndexFieldData.WithOrdinals) {
|
||||
return new BytesRefOrdValComparator((IndexFieldData.WithOrdinals<?>) indexFieldData, numHits, sortMode, missingBytes);
|
||||
}
|
||||
return new BytesRefValComparator(indexFieldData, numHits, sortMode, missingBytes);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -38,12 +38,19 @@ import java.io.IOException;
|
||||
* to large results, this comparator will be much faster
|
||||
* than {@link org.apache.lucene.search.FieldComparator.TermValComparator}. For very small
|
||||
* result sets it may be slower.
|
||||
*
|
||||
* Internally this comparator multiplies ordinals by 4 so that virtual ordinals can be inserted in-between the original field data ordinals.
|
||||
* Thanks to this, an ordinal for the missing value and the bottom value can be computed and all ordinals are directly comparable. For example,
|
||||
* if the field data ordinals are (a,1), (b,2) and (c,3), they will be internally stored as (a,4), (b,8), (c,12). Then the ordinal for the
|
||||
* missing value will be computed by binary searching. For example, if the missing value is 'ab', it will be assigned 6 as an ordinal (between
|
||||
* 'a' and 'b'. And if the bottom value is 'ac', it will be assigned 7 as an ordinal (between 'ab' and 'b').
|
||||
*/
|
||||
public final class BytesRefOrdValComparator extends FieldComparator<BytesRef> {
|
||||
public final class BytesRefOrdValComparator extends NestedWrappableComparator<BytesRef> {
|
||||
|
||||
final IndexFieldData.WithOrdinals<?> indexFieldData;
|
||||
final BytesRef missingValue;
|
||||
|
||||
/* Ords for each slot.
|
||||
/* Ords for each slot, times 4.
|
||||
@lucene.internal */
|
||||
final long[] ords;
|
||||
|
||||
@ -67,6 +74,7 @@ public final class BytesRefOrdValComparator extends FieldComparator<BytesRef> {
|
||||
/* Current reader's doc ord/values.
|
||||
@lucene.internal */
|
||||
BytesValues.WithOrdinals termsIndex;
|
||||
long missingOrd;
|
||||
|
||||
/* Bottom slot, or -1 if queue isn't full yet
|
||||
@lucene.internal */
|
||||
@ -77,21 +85,12 @@ public final class BytesRefOrdValComparator extends FieldComparator<BytesRef> {
|
||||
@lucene.internal */
|
||||
long bottomOrd;
|
||||
|
||||
/* True if current bottom slot matches the current
|
||||
reader.
|
||||
@lucene.internal */
|
||||
boolean bottomSameReader;
|
||||
|
||||
/* Bottom value (same as values[bottomSlot] once
|
||||
bottomSlot is set). Cached for faster compares.
|
||||
@lucene.internal */
|
||||
BytesRef bottomValue;
|
||||
|
||||
final BytesRef tempBR = new BytesRef();
|
||||
|
||||
public BytesRefOrdValComparator(IndexFieldData.WithOrdinals<?> indexFieldData, int numHits, SortMode sortMode) {
|
||||
public BytesRefOrdValComparator(IndexFieldData.WithOrdinals<?> indexFieldData, int numHits, SortMode sortMode, BytesRef missingValue) {
|
||||
this.indexFieldData = indexFieldData;
|
||||
this.sortMode = sortMode;
|
||||
this.missingValue = missingValue;
|
||||
ords = new long[numHits];
|
||||
values = new BytesRef[numHits];
|
||||
readerGen = new int[numHits];
|
||||
@ -121,34 +120,37 @@ public final class BytesRefOrdValComparator extends FieldComparator<BytesRef> {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottomMissing() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copy(int slot, int doc) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareDocToValue(int doc, BytesRef value) {
|
||||
BytesRef docValue = termsIndex.getValue(doc);
|
||||
if (docValue == null) {
|
||||
if (value == null) {
|
||||
return 0;
|
||||
}
|
||||
return -1;
|
||||
} else if (value == null) {
|
||||
return 1;
|
||||
}
|
||||
return docValue.compareTo(value);
|
||||
public void missing(int slot) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
/**
|
||||
* Base class for specialized (per bit width of the
|
||||
* ords) per-segment comparator. NOTE: this is messy;
|
||||
* we do this only because hotspot can't reliably inline
|
||||
* the underlying array access when looking up doc->ord
|
||||
*
|
||||
* @lucene.internal
|
||||
*/
|
||||
abstract class PerSegmentComparator extends FieldComparator<BytesRef> {
|
||||
@Override
|
||||
public int compareDocToValue(int doc, BytesRef value) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
class PerSegmentComparator extends NestedWrappableComparator<BytesRef> {
|
||||
final Ordinals.Docs readerOrds;
|
||||
final BytesValues.WithOrdinals termsIndex;
|
||||
|
||||
public PerSegmentComparator(BytesValues.WithOrdinals termsIndex) {
|
||||
this.readerOrds = termsIndex.ordinals();
|
||||
this.termsIndex = termsIndex;
|
||||
if (readerOrds.getNumOrds() > Long.MAX_VALUE / 4) {
|
||||
throw new IllegalStateException("Current terms index pretends it has more than " + (Long.MAX_VALUE / 4) + " ordinals, which is unsupported by this impl");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
|
||||
@ -185,225 +187,109 @@ public final class BytesRefOrdValComparator extends FieldComparator<BytesRef> {
|
||||
|
||||
@Override
|
||||
public int compareDocToValue(int doc, BytesRef value) {
|
||||
return BytesRefOrdValComparator.this.compareDocToValue(doc, value);
|
||||
final long ord = getOrd(doc);
|
||||
final BytesRef docValue = ord == 0 ? missingValue : termsIndex.getValueByOrd(ord);
|
||||
return compareValues(docValue, value);
|
||||
}
|
||||
}
|
||||
|
||||
// Used per-segment when bit width of doc->ord is 8:
|
||||
private final class ByteOrdComparator extends PerSegmentComparator {
|
||||
private final byte[] readerOrds;
|
||||
private final BytesValues.WithOrdinals termsIndex;
|
||||
private final int docBase;
|
||||
|
||||
public ByteOrdComparator(byte[] readerOrds, BytesValues.WithOrdinals termsIndex, int docBase) {
|
||||
this.readerOrds = readerOrds;
|
||||
this.termsIndex = termsIndex;
|
||||
this.docBase = docBase;
|
||||
protected long getOrd(int doc) {
|
||||
return readerOrds.getOrd(doc);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottom(int doc) {
|
||||
assert bottomSlot != -1;
|
||||
final int docOrd = (readerOrds[doc] & 0xFF);
|
||||
if (bottomSameReader) {
|
||||
// ord is precisely comparable, even in the equal case
|
||||
return (int) bottomOrd - docOrd;
|
||||
} else if (bottomOrd >= docOrd) {
|
||||
// the equals case always means bottom is > doc
|
||||
// (because we set bottomOrd to the lower bound in
|
||||
// setBottom):
|
||||
return 1;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
final long docOrd = getOrd(doc);
|
||||
final long comparableOrd = docOrd == 0 ? missingOrd : docOrd << 2;
|
||||
return LongValuesComparator.compare(bottomOrd, comparableOrd);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottomMissing() {
|
||||
assert bottomSlot != -1;
|
||||
return LongValuesComparator.compare(bottomOrd, missingOrd);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copy(int slot, int doc) {
|
||||
final int ord = readerOrds[doc] & 0xFF;
|
||||
ords[slot] = ord;
|
||||
final long ord = getOrd(doc);
|
||||
if (ord == 0) {
|
||||
values[slot] = null;
|
||||
ords[slot] = missingOrd;
|
||||
values[slot] = missingValue;
|
||||
} else {
|
||||
assert ord > 0;
|
||||
if (values[slot] == null) {
|
||||
ords[slot] = ord << 2;
|
||||
if (values[slot] == null || values[slot] == missingValue) {
|
||||
values[slot] = new BytesRef();
|
||||
}
|
||||
termsIndex.getValueScratchByOrd(ord, values[slot]);
|
||||
}
|
||||
readerGen[slot] = currentReaderGen;
|
||||
}
|
||||
}
|
||||
|
||||
// Used per-segment when bit width of doc->ord is 16:
|
||||
private final class ShortOrdComparator extends PerSegmentComparator {
|
||||
private final short[] readerOrds;
|
||||
private final BytesValues.WithOrdinals termsIndex;
|
||||
private final int docBase;
|
||||
|
||||
public ShortOrdComparator(short[] readerOrds, BytesValues.WithOrdinals termsIndex, int docBase) {
|
||||
this.readerOrds = readerOrds;
|
||||
this.termsIndex = termsIndex;
|
||||
this.docBase = docBase;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottom(int doc) {
|
||||
assert bottomSlot != -1;
|
||||
final int docOrd = (readerOrds[doc] & 0xFFFF);
|
||||
if (bottomSameReader) {
|
||||
// ord is precisely comparable, even in the equal case
|
||||
return (int) bottomOrd - docOrd;
|
||||
} else if (bottomOrd >= docOrd) {
|
||||
// the equals case always means bottom is > doc
|
||||
// (because we set bottomOrd to the lower bound in
|
||||
// setBottom):
|
||||
return 1;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copy(int slot, int doc) {
|
||||
final int ord = readerOrds[doc] & 0xFFFF;
|
||||
ords[slot] = ord;
|
||||
if (ord == 0) {
|
||||
values[slot] = null;
|
||||
} else {
|
||||
assert ord > 0;
|
||||
if (values[slot] == null) {
|
||||
values[slot] = new BytesRef();
|
||||
}
|
||||
termsIndex.getValueScratchByOrd(ord, values[slot]);
|
||||
}
|
||||
readerGen[slot] = currentReaderGen;
|
||||
public void missing(int slot) {
|
||||
ords[slot] = missingOrd;
|
||||
values[slot] = missingValue;
|
||||
}
|
||||
}
|
||||
|
||||
// Used per-segment when bit width of doc->ord is 32:
|
||||
private final class IntOrdComparator extends PerSegmentComparator {
|
||||
private final int[] readerOrds;
|
||||
private final BytesValues.WithOrdinals termsIndex;
|
||||
private final int docBase;
|
||||
|
||||
public IntOrdComparator(int[] readerOrds, BytesValues.WithOrdinals termsIndex, int docBase) {
|
||||
this.readerOrds = readerOrds;
|
||||
this.termsIndex = termsIndex;
|
||||
this.docBase = docBase;
|
||||
// for assertions
|
||||
private boolean consistentInsertedOrd(BytesValues.WithOrdinals termsIndex, long ord, BytesRef value) {
|
||||
assert ord >= 0 : ord;
|
||||
assert (ord == 0) == (value == null) : "ord=" + ord + ", value=" + value;
|
||||
final long previousOrd = ord >>> 2;
|
||||
final long nextOrd = previousOrd + 1;
|
||||
final BytesRef previous = previousOrd == 0 ? null : termsIndex.getValueByOrd(previousOrd);
|
||||
if ((ord & 3) == 0) { // there was an existing ord with the inserted value
|
||||
assert compareValues(previous, value) == 0;
|
||||
} else {
|
||||
assert compareValues(previous, value) < 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottom(int doc) {
|
||||
assert bottomSlot != -1;
|
||||
final int docOrd = readerOrds[doc];
|
||||
if (bottomSameReader) {
|
||||
// ord is precisely comparable, even in the equal case
|
||||
return (int) bottomOrd - docOrd;
|
||||
} else if (bottomOrd >= docOrd) {
|
||||
// the equals case always means bottom is > doc
|
||||
// (because we set bottomOrd to the lower bound in
|
||||
// setBottom):
|
||||
return 1;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copy(int slot, int doc) {
|
||||
final int ord = readerOrds[doc];
|
||||
ords[slot] = ord;
|
||||
if (ord == 0) {
|
||||
values[slot] = null;
|
||||
} else {
|
||||
assert ord > 0;
|
||||
if (values[slot] == null) {
|
||||
values[slot] = new BytesRef();
|
||||
}
|
||||
termsIndex.getValueScratchByOrd(ord, values[slot]);
|
||||
}
|
||||
readerGen[slot] = currentReaderGen;
|
||||
if (nextOrd < termsIndex.ordinals().getMaxOrd()) {
|
||||
final BytesRef next = termsIndex.getValueByOrd(nextOrd);
|
||||
assert compareValues(value, next) < 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Used per-segment when bit width is not a native array
|
||||
// size (8, 16, 32):
|
||||
final class AnyOrdComparator extends PerSegmentComparator {
|
||||
private final IndexFieldData fieldData;
|
||||
private final Ordinals.Docs readerOrds;
|
||||
private final BytesValues.WithOrdinals termsIndex;
|
||||
private final int docBase;
|
||||
|
||||
public AnyOrdComparator(IndexFieldData fieldData, BytesValues.WithOrdinals termsIndex, int docBase) {
|
||||
this.fieldData = fieldData;
|
||||
this.readerOrds = termsIndex.ordinals();
|
||||
this.termsIndex = termsIndex;
|
||||
this.docBase = docBase;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottom(int doc) {
|
||||
assert bottomSlot != -1;
|
||||
final long docOrd = readerOrds.getOrd(doc);
|
||||
if (bottomSameReader) {
|
||||
// ord is precisely comparable, even in the equal case
|
||||
return LongValuesComparator.compare(bottomOrd, docOrd);
|
||||
} else if (bottomOrd >= docOrd) {
|
||||
// the equals case always means bottom is > doc
|
||||
// (because we set bottomOrd to the lower bound in
|
||||
// setBottom):
|
||||
return 1;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copy(int slot, int doc) {
|
||||
final long ord = readerOrds.getOrd(doc);
|
||||
ords[slot] = ord;
|
||||
if (ord == 0) {
|
||||
values[slot] = null;
|
||||
} else {
|
||||
assert ord > 0;
|
||||
if (values[slot] == null) {
|
||||
values[slot] = new BytesRef();
|
||||
}
|
||||
termsIndex.getValueScratchByOrd(ord, values[slot]);
|
||||
}
|
||||
readerGen[slot] = currentReaderGen;
|
||||
// find where to insert an ord in the current terms index
|
||||
private long ordInCurrentReader(BytesValues.WithOrdinals termsIndex, BytesRef value) {
|
||||
final long docOrd = binarySearch(termsIndex, value);
|
||||
assert docOrd != -1; // would mean smaller than null
|
||||
final long ord;
|
||||
if (docOrd >= 0) {
|
||||
// value exists in the current segment
|
||||
ord = docOrd << 2;
|
||||
} else {
|
||||
// value doesn't exist, use the ord between the previous and the next term
|
||||
ord = ((-2 - docOrd) << 2) + 2;
|
||||
}
|
||||
assert (ord & 1) == 0;
|
||||
return ord;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
|
||||
final int docBase = context.docBase;
|
||||
termsIndex = indexFieldData.load(context).getBytesValues();
|
||||
assert termsIndex.ordinals() != null && termsIndex.ordinals().ordinals() != null;
|
||||
if (missingValue == null) {
|
||||
missingOrd = 0;
|
||||
} else {
|
||||
missingOrd = ordInCurrentReader(termsIndex, missingValue);
|
||||
assert consistentInsertedOrd(termsIndex, missingOrd, missingValue);
|
||||
}
|
||||
FieldComparator<BytesRef> perSegComp = null;
|
||||
assert termsIndex.ordinals() != null && termsIndex.ordinals().ordinals() != null;
|
||||
if (termsIndex.isMultiValued()) {
|
||||
perSegComp = new MultiAnyOrdComparator(termsIndex);
|
||||
} else {
|
||||
final Ordinals.Docs docToOrd = termsIndex.ordinals();
|
||||
if (docToOrd.ordinals().hasSingleArrayBackingStorage()) {
|
||||
Object ordsStorage = docToOrd.ordinals().getBackingStorage();
|
||||
if (ordsStorage instanceof byte[]) {
|
||||
perSegComp = new ByteOrdComparator((byte[]) ordsStorage, termsIndex, docBase);
|
||||
} else if (ordsStorage instanceof short[]) {
|
||||
perSegComp = new ShortOrdComparator((short[]) ordsStorage, termsIndex, docBase);
|
||||
} else if (ordsStorage instanceof int[]) {
|
||||
perSegComp = new IntOrdComparator((int[]) ordsStorage, termsIndex, docBase);
|
||||
perSegComp = new PerSegmentComparator(termsIndex) {
|
||||
@Override
|
||||
protected long getOrd(int doc) {
|
||||
return getRelevantOrd(readerOrds, doc, sortMode);
|
||||
}
|
||||
}
|
||||
// Don't specialize the long[] case since it's not
|
||||
// possible, ie, worse case is MAX_INT-1 docs with
|
||||
// every one having a unique value.
|
||||
|
||||
// TODO: ES - should we optimize for the PackedInts.Reader case as well?
|
||||
if (perSegComp == null) {
|
||||
perSegComp = new AnyOrdComparator(indexFieldData, termsIndex, docBase);
|
||||
}
|
||||
};
|
||||
} else {
|
||||
perSegComp = new PerSegmentComparator(termsIndex);
|
||||
}
|
||||
currentReaderGen++;
|
||||
if (bottomSlot != -1) {
|
||||
@ -415,32 +301,29 @@ public final class BytesRefOrdValComparator extends FieldComparator<BytesRef> {
|
||||
@Override
|
||||
public void setBottom(final int bottom) {
|
||||
bottomSlot = bottom;
|
||||
final BytesRef bottomValue = values[bottomSlot];
|
||||
|
||||
bottomValue = values[bottomSlot];
|
||||
if (currentReaderGen == readerGen[bottomSlot]) {
|
||||
if (bottomValue == null) {
|
||||
bottomOrd = 0;
|
||||
} else if (currentReaderGen == readerGen[bottomSlot]) {
|
||||
bottomOrd = ords[bottomSlot];
|
||||
bottomSameReader = true;
|
||||
} else {
|
||||
if (bottomValue == null) {
|
||||
// 0 ord is null for all segments
|
||||
assert ords[bottomSlot] == 0;
|
||||
bottomOrd = 0;
|
||||
bottomSameReader = true;
|
||||
readerGen[bottomSlot] = currentReaderGen;
|
||||
} else {
|
||||
final long index = binarySearch(termsIndex, bottomValue);
|
||||
if (index < 0) {
|
||||
bottomOrd = -index - 2;
|
||||
bottomSameReader = false;
|
||||
} else {
|
||||
bottomOrd = index;
|
||||
// exact value match
|
||||
bottomSameReader = true;
|
||||
readerGen[bottomSlot] = currentReaderGen;
|
||||
ords[bottomSlot] = bottomOrd;
|
||||
// insert an ord
|
||||
bottomOrd = ordInCurrentReader(termsIndex, bottomValue);
|
||||
if (bottomOrd == missingOrd) {
|
||||
// bottomValue and missingValue and in-between the same field data values -> tie-break
|
||||
// this is why we multiply ords by 4
|
||||
assert missingValue != null;
|
||||
final int cmp = bottomValue.compareTo(missingValue);
|
||||
if (cmp < 0) {
|
||||
--bottomOrd;
|
||||
} else if (cmp > 0) {
|
||||
++bottomOrd;
|
||||
}
|
||||
}
|
||||
assert consistentInsertedOrd(termsIndex, bottomOrd, bottomValue);
|
||||
}
|
||||
readerGen[bottomSlot] = currentReaderGen;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -454,7 +337,7 @@ public final class BytesRefOrdValComparator extends FieldComparator<BytesRef> {
|
||||
|
||||
final protected static long binarySearch(BytesValues.WithOrdinals a, BytesRef key, long low, long high) {
|
||||
assert a.getValueByOrd(high) == null | a.getValueByOrd(high) != null; // make sure we actually can get these values
|
||||
assert a.getValueByOrd(low) == null | a.getValueByOrd(low) != null;
|
||||
assert low == high + 1 || a.getValueByOrd(low) == null | a.getValueByOrd(low) != null;
|
||||
while (low <= high) {
|
||||
long mid = (low + high) >>> 1;
|
||||
BytesRef midVal = a.getValueByOrd(mid);
|
||||
@ -475,65 +358,6 @@ public final class BytesRefOrdValComparator extends FieldComparator<BytesRef> {
|
||||
return -(low + 1);
|
||||
}
|
||||
|
||||
|
||||
class MultiAnyOrdComparator extends PerSegmentComparator {
|
||||
|
||||
private final BytesValues.WithOrdinals termsIndex;
|
||||
private final Ordinals.Docs readerOrds;
|
||||
|
||||
private MultiAnyOrdComparator(BytesValues.WithOrdinals termsIndex) {
|
||||
this.termsIndex = termsIndex;
|
||||
this.readerOrds = termsIndex.ordinals();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottom(int doc) throws IOException {
|
||||
final long docOrd = getRelevantOrd(readerOrds, doc, sortMode);
|
||||
if (bottomSameReader) {
|
||||
// ord is precisely comparable, even in the equal case
|
||||
return LongValuesComparator.compare(bottomOrd, docOrd);
|
||||
} else if (bottomOrd >= docOrd) {
|
||||
// the equals case always means bottom is > doc
|
||||
// (because we set bottomOrd to the lower bound in
|
||||
// setBottom):
|
||||
return 1;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copy(int slot, int doc) throws IOException {
|
||||
final long ord = getRelevantOrd(readerOrds, doc, sortMode);
|
||||
ords[slot] = ord;
|
||||
if (ord == 0) {
|
||||
values[slot] = null;
|
||||
} else {
|
||||
assert ord > 0;
|
||||
if (values[slot] == null) {
|
||||
values[slot] = new BytesRef();
|
||||
}
|
||||
termsIndex.getValueScratchByOrd(ord, values[slot]);
|
||||
}
|
||||
readerGen[slot] = currentReaderGen;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareDocToValue(int doc, BytesRef value) {
|
||||
BytesRef docValue = getRelevantValue(termsIndex, doc, sortMode);
|
||||
if (docValue == null) {
|
||||
if (value == null) {
|
||||
return 0;
|
||||
}
|
||||
return -1;
|
||||
} else if (value == null) {
|
||||
return 1;
|
||||
}
|
||||
return docValue.compareTo(value);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static BytesRef getRelevantValue(BytesValues.WithOrdinals readerValues, int docId, SortMode sortMode) {
|
||||
BytesValues.Iter iter = readerValues.getIter(docId);
|
||||
if (!iter.hasNext()) {
|
||||
|
||||
@ -19,33 +19,37 @@
|
||||
|
||||
package org.elasticsearch.index.fielddata.fieldcomparator;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.search.FieldComparator;
|
||||
import org.apache.lucene.util.ArrayUtil;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.RamUsageEstimator;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Sorts by field's natural Term sort order. All
|
||||
* comparisons are done using BytesRef.compareTo, which is
|
||||
* slow for medium to large result sets but possibly
|
||||
* very fast for very small results sets.
|
||||
*/
|
||||
public final class BytesRefValComparator extends FieldComparator<BytesRef> {
|
||||
public final class BytesRefValComparator extends NestedWrappableComparator<BytesRef> {
|
||||
|
||||
private final IndexFieldData<?> indexFieldData;
|
||||
private final SortMode sortMode;
|
||||
private final BytesRef missingValue;
|
||||
|
||||
private final BytesRef[] values;
|
||||
private BytesRef bottom;
|
||||
private BytesValues docTerms;
|
||||
|
||||
BytesRefValComparator(IndexFieldData<?> indexFieldData, int numHits, SortMode sortMode) {
|
||||
BytesRefValComparator(IndexFieldData<?> indexFieldData, int numHits, SortMode sortMode, BytesRef missingValue) {
|
||||
this.sortMode = sortMode;
|
||||
values = new BytesRef[numHits];
|
||||
this.indexFieldData = indexFieldData;
|
||||
this.missingValue = missingValue;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -57,16 +61,23 @@ public final class BytesRefValComparator extends FieldComparator<BytesRef> {
|
||||
|
||||
@Override
|
||||
public int compareBottom(int doc) throws IOException {
|
||||
final BytesRef val2 = docTerms.getValue(doc);
|
||||
BytesRef val2 = docTerms.getValue(doc);
|
||||
if (val2 == null) {
|
||||
val2 = missingValue;
|
||||
}
|
||||
return compareValues(bottom, val2);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void copy(int slot, int doc) throws IOException {
|
||||
if (values[slot] == null) {
|
||||
values[slot] = new BytesRef();
|
||||
if (!docTerms.hasValue(doc)) {
|
||||
values[slot] = missingValue;
|
||||
} else {
|
||||
if (values[slot] == null || values[slot] == missingValue) {
|
||||
values[slot] = new BytesRef();
|
||||
}
|
||||
docTerms.getValueScratch(doc, values[slot]);
|
||||
}
|
||||
docTerms.getValueScratch(doc, values[slot]);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -143,23 +154,32 @@ public final class BytesRefValComparator extends FieldComparator<BytesRef> {
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueScratch(int docId, BytesRef scratch) {
|
||||
public BytesRef getValueScratch(int docId, BytesRef relevantVal) {
|
||||
BytesValues.Iter iter = delegate.getIter(docId);
|
||||
if (!iter.hasNext()) {
|
||||
return null;
|
||||
relevantVal.length = 0;
|
||||
return relevantVal;
|
||||
}
|
||||
|
||||
BytesRef currentVal = iter.next();
|
||||
BytesRef relevantVal = currentVal;
|
||||
// We MUST allocate a new byte[] since relevantVal might have been filled by reference by a PagedBytes instance
|
||||
// meaning that the BytesRef.bytes are shared and shouldn't be overwritten. We can't use the bytes of the iterator
|
||||
// either because they will be overwritten by subsequent calls in the current thread
|
||||
relevantVal.bytes = new byte[ArrayUtil.oversize(currentVal.length, RamUsageEstimator.NUM_BYTES_BYTE)];
|
||||
relevantVal.offset = 0;
|
||||
relevantVal.length = 0;
|
||||
relevantVal.append(currentVal);
|
||||
while (true) {
|
||||
int cmp = currentVal.compareTo(relevantVal);
|
||||
if (sortMode == SortMode.MAX) {
|
||||
if (cmp > 0) {
|
||||
relevantVal = currentVal;
|
||||
relevantVal.length = 0;
|
||||
relevantVal.append(currentVal);
|
||||
}
|
||||
} else {
|
||||
if (cmp < 0) {
|
||||
relevantVal = currentVal;
|
||||
relevantVal.length = 0;
|
||||
relevantVal.append(currentVal);
|
||||
}
|
||||
}
|
||||
if (!iter.hasNext()) {
|
||||
@ -172,4 +192,14 @@ public final class BytesRefValComparator extends FieldComparator<BytesRef> {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void missing(int slot) {
|
||||
values[slot] = missingValue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareBottomMissing() {
|
||||
return compareValues(bottom, missingValue);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.fielddata.fieldcomparator;
|
||||
|
||||
import org.apache.lucene.search.FieldComparator;
|
||||
|
||||
/** Base comparator which allows for nested sorting. */
|
||||
public abstract class NestedWrappableComparator<T> extends FieldComparator<T> {
|
||||
|
||||
/**
|
||||
* Assigns the underlying missing value to the specified slot, if the actual implementation supports missing value.
|
||||
*
|
||||
* @param slot The slot to assign the the missing value to.
|
||||
*/
|
||||
public abstract void missing(int slot);
|
||||
|
||||
/**
|
||||
* Compares the missing value to the bottom.
|
||||
*
|
||||
* @return any N < 0 if the bottom value is not competitive with the missing value, any N > 0 if the
|
||||
* bottom value is competitive with the missing value and 0 if they are equal.
|
||||
*/
|
||||
public abstract int compareBottomMissing();
|
||||
|
||||
}
|
||||
@ -19,13 +19,12 @@
|
||||
|
||||
package org.elasticsearch.index.fielddata.fieldcomparator;
|
||||
|
||||
import org.apache.lucene.search.FieldComparator;
|
||||
|
||||
/**
|
||||
* Base FieldComparator class for number fields.
|
||||
*/
|
||||
// This is right now only used for sorting number based fields inside nested objects
|
||||
public abstract class NumberComparatorBase<T> extends FieldComparator<T> {
|
||||
public abstract class NumberComparatorBase<T> extends NestedWrappableComparator<T> {
|
||||
|
||||
/**
|
||||
* Adds numeric value at the specified doc to the specified slot.
|
||||
@ -43,18 +42,4 @@ public abstract class NumberComparatorBase<T> extends FieldComparator<T> {
|
||||
*/
|
||||
public abstract void divide(int slot, int divisor);
|
||||
|
||||
/**
|
||||
* Assigns the underlying missing value to the specified slot, if the actual implementation supports missing value.
|
||||
*
|
||||
* @param slot The slot to assign the the missing value to.
|
||||
*/
|
||||
public abstract void missing(int slot);
|
||||
|
||||
/**
|
||||
* Compares the missing value to the bottom.
|
||||
*
|
||||
* @return any N < 0 if the bottom value is not competitive with the missing value, any N > 0 if the
|
||||
* bottom value is competitive with the missing value and 0 if they are equal.
|
||||
*/
|
||||
public abstract int compareBottomMissing();
|
||||
}
|
||||
|
||||
@ -59,8 +59,7 @@ public abstract class AbstractBytesIndexFieldData<FD extends AtomicFieldData.Wit
|
||||
|
||||
@Override
|
||||
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, SortMode sortMode) {
|
||||
// TODO support "missingValue" for sortMissingValue options here...
|
||||
return new BytesRefFieldComparatorSource(this, sortMode);
|
||||
return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
|
||||
}
|
||||
|
||||
protected TermsEnum filter(Terms terms, AtomicReader reader) throws IOException {
|
||||
|
||||
@ -28,6 +28,7 @@ import org.apache.lucene.util.FixedBitSet;
|
||||
import org.elasticsearch.ElasticSearchIllegalArgumentException;
|
||||
import org.elasticsearch.common.lucene.docset.DocIdSets;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.NestedWrappableComparator;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.NumberComparatorBase;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.SortMode;
|
||||
|
||||
@ -60,9 +61,9 @@ public class NestedFieldComparatorSource extends IndexFieldData.XFieldComparator
|
||||
case MIN:
|
||||
return new NestedFieldComparator.Lowest(wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
|
||||
case SUM:
|
||||
return new NestedFieldComparator.Sum((NumberComparatorBase) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
|
||||
return new NestedFieldComparator.Sum((NumberComparatorBase<?>) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
|
||||
case AVG:
|
||||
return new NestedFieldComparator.Avg((NumberComparatorBase) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
|
||||
return new NestedFieldComparator.Avg((NumberComparatorBase<?>) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits);
|
||||
default:
|
||||
throw new ElasticSearchIllegalArgumentException(
|
||||
String.format(Locale.ROOT, "Unsupported sort_mode[%s] for nested type", sortMode)
|
||||
@ -194,7 +195,6 @@ abstract class NestedFieldComparator extends FieldComparator {
|
||||
copyMissing(wrappedComparator, slot);
|
||||
return;
|
||||
}
|
||||
wrappedComparator.copy(spareSlot, nestedDoc);
|
||||
wrappedComparator.copy(slot, nestedDoc);
|
||||
|
||||
while (true) {
|
||||
@ -263,7 +263,6 @@ abstract class NestedFieldComparator extends FieldComparator {
|
||||
copyMissing(wrappedComparator, slot);
|
||||
return;
|
||||
}
|
||||
wrappedComparator.copy(spareSlot, nestedDoc);
|
||||
wrappedComparator.copy(slot, nestedDoc);
|
||||
|
||||
while (true) {
|
||||
@ -406,15 +405,15 @@ abstract class NestedFieldComparator extends FieldComparator {
|
||||
}
|
||||
}
|
||||
|
||||
static final void copyMissing(FieldComparator comparator, int slot) {
|
||||
if (comparator instanceof NumberComparatorBase) {
|
||||
((NumberComparatorBase) comparator).missing(slot);
|
||||
static final void copyMissing(FieldComparator<?> comparator, int slot) {
|
||||
if (comparator instanceof NestedWrappableComparator<?>) {
|
||||
((NestedWrappableComparator<?>) comparator).missing(slot);
|
||||
}
|
||||
}
|
||||
|
||||
static final int compareBottomMissing(FieldComparator comparator) {
|
||||
if (comparator instanceof NumberComparatorBase) {
|
||||
return ((NumberComparatorBase) comparator).compareBottomMissing();
|
||||
static final int compareBottomMissing(FieldComparator<?> comparator) {
|
||||
if (comparator instanceof NestedWrappableComparator<?>) {
|
||||
return ((NestedWrappableComparator<?>) comparator).compareBottomMissing();
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -35,6 +35,7 @@ import org.elasticsearch.common.text.Text;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilderString;
|
||||
import org.elasticsearch.common.xcontent.XContentHelper;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
|
||||
import org.elasticsearch.rest.action.support.RestXContentBuilder;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import org.elasticsearch.search.SearchHitField;
|
||||
@ -58,6 +59,7 @@ import static org.elasticsearch.search.internal.InternalSearchHitField.readSearc
|
||||
public class InternalSearchHit implements SearchHit {
|
||||
|
||||
private static final Object[] EMPTY_SORT_VALUES = new Object[0];
|
||||
private static final Text MAX_TERM_AS_TEXT = new StringAndBytesText(BytesRefFieldComparatorSource.MAX_TERM.utf8ToString());
|
||||
|
||||
private transient int docId;
|
||||
|
||||
@ -444,7 +446,13 @@ public class InternalSearchHit implements SearchHit {
|
||||
if (sortValues != null && sortValues.length > 0) {
|
||||
builder.startArray(Fields.SORT);
|
||||
for (Object sortValue : sortValues) {
|
||||
builder.value(sortValue);
|
||||
if (sortValue != null && sortValue.equals(MAX_TERM_AS_TEXT)) {
|
||||
// We don't display MAX_TERM in JSON responses in case some clients have UTF-8 parsers that wouldn't accept a
|
||||
// non-character in the response, even though this is valid UTF-8
|
||||
builder.nullValue();
|
||||
} else {
|
||||
builder.value(sortValue);
|
||||
}
|
||||
}
|
||||
builder.endArray();
|
||||
}
|
||||
|
||||
@ -790,9 +790,9 @@ public class SimpleSortTests extends AbstractSharedClusterTest {
|
||||
assertNoFailures(searchResponse);
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
|
||||
assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("2"));
|
||||
assertThat((String) searchResponse.getHits().getAt(1).field("id").value(), equalTo("1"));
|
||||
assertThat((String) searchResponse.getHits().getAt(2).field("id").value(), equalTo("3"));
|
||||
assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("1"));
|
||||
assertThat((String) searchResponse.getHits().getAt(1).field("id").value(), equalTo("3"));
|
||||
assertThat((String) searchResponse.getHits().getAt(2).field("id").value(), equalTo("2"));
|
||||
|
||||
searchResponse = client().prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
@ -803,9 +803,9 @@ public class SimpleSortTests extends AbstractSharedClusterTest {
|
||||
assertNoFailures(searchResponse);
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
|
||||
assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("2"));
|
||||
assertThat((String) searchResponse.getHits().getAt(1).field("id").value(), equalTo("1"));
|
||||
assertThat((String) searchResponse.getHits().getAt(2).field("id").value(), equalTo("3"));
|
||||
assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("1"));
|
||||
assertThat((String) searchResponse.getHits().getAt(1).field("id").value(), equalTo("3"));
|
||||
assertThat((String) searchResponse.getHits().getAt(2).field("id").value(), equalTo("2"));
|
||||
|
||||
searchResponse = client().prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
@ -846,7 +846,7 @@ public class SimpleSortTests extends AbstractSharedClusterTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSortMissing() throws Exception {
|
||||
public void testSortMissingNumbers() throws Exception {
|
||||
prepareCreate("test").addMapping("type1",
|
||||
XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
@ -917,6 +917,90 @@ public class SimpleSortTests extends AbstractSharedClusterTest {
|
||||
assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSortMissingStrings() throws ElasticSearchException, IOException {
|
||||
prepareCreate("test").addMapping("type1",
|
||||
XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.startObject("type1")
|
||||
.startObject("properties")
|
||||
.startObject("value")
|
||||
.field("type", "string")
|
||||
.field("index", "not_analyzed")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()).execute().actionGet();
|
||||
ensureGreen();
|
||||
client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
|
||||
.field("id", "1")
|
||||
.field("value", "a")
|
||||
.endObject()).execute().actionGet();
|
||||
|
||||
client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject()
|
||||
.field("id", "2")
|
||||
.endObject()).execute().actionGet();
|
||||
|
||||
client().prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject()
|
||||
.field("id", "1")
|
||||
.field("value", "c")
|
||||
.endObject()).execute().actionGet();
|
||||
|
||||
client().admin().indices().prepareFlush().setRefresh(true).execute().actionGet();try {
|
||||
Thread.sleep(2000);
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException();
|
||||
}
|
||||
|
||||
logger.info("--> sort with no missing (same as missing _last)");
|
||||
SearchResponse searchResponse = client().prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC))
|
||||
.execute().actionGet();
|
||||
assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0));
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3"));
|
||||
assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2"));
|
||||
|
||||
logger.info("--> sort with missing _last");
|
||||
searchResponse = client().prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("_last"))
|
||||
.execute().actionGet();
|
||||
assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0));
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3"));
|
||||
assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2"));
|
||||
|
||||
logger.info("--> sort with missing _first");
|
||||
searchResponse = client().prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("_first"))
|
||||
.execute().actionGet();
|
||||
assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0));
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().getAt(0).id(), equalTo("2"));
|
||||
assertThat(searchResponse.getHits().getAt(1).id(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3"));
|
||||
|
||||
logger.info("--> sort with missing b");
|
||||
searchResponse = client().prepareSearch()
|
||||
.setQuery(matchAllQuery())
|
||||
.addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("b"))
|
||||
.execute().actionGet();
|
||||
assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0));
|
||||
|
||||
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
|
||||
assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
|
||||
assertThat(searchResponse.getHits().getAt(1).id(), equalTo("2"));
|
||||
assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIgnoreUnmapped() throws Exception {
|
||||
createIndex("test");
|
||||
|
||||
@ -0,0 +1,460 @@
|
||||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.test.unit.index.fielddata;
|
||||
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.common.lucene.HashedBytesRef;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.SortMode;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.hamcrest.Matchers.*;
|
||||
|
||||
public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests {
|
||||
|
||||
protected String one() {
|
||||
return "1";
|
||||
}
|
||||
|
||||
protected String two() {
|
||||
return "2";
|
||||
}
|
||||
|
||||
protected String three() {
|
||||
return "3";
|
||||
}
|
||||
|
||||
protected String four() {
|
||||
return "4";
|
||||
}
|
||||
|
||||
protected String toString(Object value) {
|
||||
if (value instanceof BytesRef) {
|
||||
return ((BytesRef) value).utf8ToString();
|
||||
}
|
||||
return value.toString();
|
||||
}
|
||||
|
||||
protected abstract void fillSingleValueAllSet() throws Exception;
|
||||
|
||||
protected abstract void add2SingleValuedDocumentsAndDeleteOneOfThem() throws Exception;
|
||||
|
||||
@Test
|
||||
public void testDeletedDocs() throws Exception {
|
||||
add2SingleValuedDocumentsAndDeleteOneOfThem();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicReaderContext readerContext = refreshReader();
|
||||
AtomicFieldData fieldData = indexFieldData.load(readerContext);
|
||||
BytesValues values = fieldData.getBytesValues();
|
||||
for (int i = 0; i < fieldData.getNumDocs(); ++i) {
|
||||
assertThat(values.hasValue(i), equalTo(true));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingleValueAllSet() throws Exception {
|
||||
fillSingleValueAllSet();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicReaderContext readerContext = refreshReader();
|
||||
AtomicFieldData fieldData = indexFieldData.load(readerContext);
|
||||
assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
|
||||
|
||||
assertThat(fieldData.getNumDocs(), equalTo(3));
|
||||
|
||||
BytesValues bytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(bytesValues.isMultiValued(), equalTo(false));
|
||||
|
||||
assertThat(bytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(1), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
|
||||
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
|
||||
|
||||
BytesRef bytesRef = new BytesRef();
|
||||
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(one())));
|
||||
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(three())));
|
||||
|
||||
|
||||
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
BytesValues hashedBytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(1), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(one())));
|
||||
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
|
||||
|
||||
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
IndexSearcher searcher = new IndexSearcher(readerContext.reader());
|
||||
TopFieldDocs topDocs;
|
||||
|
||||
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
|
||||
new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN))));
|
||||
assertThat(topDocs.totalHits, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
|
||||
assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one()));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
|
||||
assertThat(toString(((FieldDoc) topDocs.scoreDocs[1]).fields[0]), equalTo(two()));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
|
||||
assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three()));
|
||||
|
||||
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
|
||||
new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true)));
|
||||
assertThat(topDocs.totalHits, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
|
||||
}
|
||||
|
||||
private HashedBytesRef convert(BytesValues values, int doc) {
|
||||
BytesRef ref = new BytesRef();
|
||||
return new HashedBytesRef(ref, values.getValueHashed(doc, ref));
|
||||
}
|
||||
|
||||
protected abstract void fillSingleValueWithMissing() throws Exception;
|
||||
|
||||
@Test
|
||||
public void testSingleValueWithMissing() throws Exception {
|
||||
fillSingleValueWithMissing();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
|
||||
assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
|
||||
|
||||
assertThat(fieldData.getNumDocs(), equalTo(3));
|
||||
|
||||
BytesValues bytesValues = fieldData
|
||||
.getBytesValues();
|
||||
|
||||
assertThat(bytesValues.isMultiValued(), equalTo(false));
|
||||
|
||||
assertThat(bytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(bytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValue(1), nullValue());
|
||||
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
|
||||
|
||||
BytesRef bytesRef = new BytesRef();
|
||||
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
|
||||
assertThat(bytesRef, equalTo(new BytesRef()));
|
||||
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(three())));
|
||||
|
||||
|
||||
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
bytesValuesIter = bytesValues.getIter(1);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
BytesValues hashedBytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(new BytesRef())));
|
||||
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
|
||||
|
||||
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
hashedBytesValuesIter = hashedBytesValues.getIter(1);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
}
|
||||
|
||||
protected abstract void fillMultiValueAllSet() throws Exception;
|
||||
|
||||
@Test
|
||||
public void testMultiValueAllSet() throws Exception {
|
||||
fillMultiValueAllSet();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
|
||||
assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
|
||||
|
||||
assertThat(fieldData.getNumDocs(), equalTo(3));
|
||||
|
||||
BytesValues bytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(bytesValues.isMultiValued(), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(1), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
|
||||
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
|
||||
|
||||
BytesRef bytesRef = new BytesRef();
|
||||
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(one())));
|
||||
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(three())));
|
||||
|
||||
|
||||
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
BytesValues hashedBytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(1), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(one())));
|
||||
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
|
||||
|
||||
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(four())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
|
||||
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN))));
|
||||
assertThat(topDocs.totalHits, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs.length, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
|
||||
|
||||
topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true)));
|
||||
assertThat(topDocs.totalHits, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs.length, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
|
||||
}
|
||||
|
||||
protected abstract void fillMultiValueWithMissing() throws Exception;
|
||||
|
||||
@Test
|
||||
public void testMultiValueWithMissing() throws Exception {
|
||||
fillMultiValueWithMissing();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
|
||||
assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
|
||||
|
||||
assertThat(fieldData.getNumDocs(), equalTo(3));
|
||||
|
||||
BytesValues bytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(bytesValues.isMultiValued(), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(bytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValue(1), nullValue());
|
||||
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
|
||||
|
||||
BytesRef bytesRef = new BytesRef();
|
||||
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
|
||||
assertThat(bytesRef, equalTo(new BytesRef()));
|
||||
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(three())));
|
||||
|
||||
|
||||
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
bytesValuesIter = bytesValues.getIter(1);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
BytesValues hashedBytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(new BytesRef())));
|
||||
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
|
||||
|
||||
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(four())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
hashedBytesValuesIter = hashedBytesValues.getIter(1);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
}
|
||||
|
||||
public void testMissingValueForAll() throws Exception {
|
||||
fillAllMissing();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
|
||||
// Some impls (FST) return size 0 and some (PagedBytes) do take size in the case no actual data is loaded
|
||||
assertThat(fieldData.getMemorySizeInBytes(), greaterThanOrEqualTo(0l));
|
||||
|
||||
assertThat(fieldData.getNumDocs(), equalTo(3));
|
||||
|
||||
BytesValues bytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(bytesValues.isMultiValued(), equalTo(false));
|
||||
|
||||
assertThat(bytesValues.hasValue(0), equalTo(false));
|
||||
assertThat(bytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(bytesValues.hasValue(2), equalTo(false));
|
||||
|
||||
assertThat(bytesValues.getValue(0), nullValue());
|
||||
assertThat(bytesValues.getValue(1), nullValue());
|
||||
assertThat(bytesValues.getValue(2), nullValue());
|
||||
|
||||
BytesRef bytesRef = new BytesRef();
|
||||
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef()));
|
||||
assertThat(bytesRef, equalTo(new BytesRef()));
|
||||
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
|
||||
assertThat(bytesRef, equalTo(new BytesRef()));
|
||||
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef()));
|
||||
assertThat(bytesRef, equalTo(new BytesRef()));
|
||||
|
||||
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
bytesValuesIter = bytesValues.getIter(1);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
bytesValuesIter = bytesValues.getIter(2);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
BytesValues hashedBytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(hashedBytesValues.hasValue(0), equalTo(false));
|
||||
assertThat(hashedBytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(hashedBytesValues.hasValue(2), equalTo(false));
|
||||
|
||||
assertThat(hashedBytesValues.getValue(0), nullValue());
|
||||
assertThat(hashedBytesValues.getValue(1), nullValue());
|
||||
assertThat(hashedBytesValues.getValue(2), nullValue());
|
||||
|
||||
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
hashedBytesValuesIter = hashedBytesValues.getIter(1);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
hashedBytesValuesIter = hashedBytesValues.getIter(2);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
}
|
||||
|
||||
protected abstract void fillAllMissing() throws Exception;
|
||||
|
||||
@Test
|
||||
public void testSortMultiValuesFields() throws Exception {
|
||||
fillExtendedMvSet();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
|
||||
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
|
||||
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10,
|
||||
new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN))));
|
||||
assertThat(topDocs.totalHits, equalTo(8));
|
||||
assertThat(topDocs.scoreDocs.length, equalTo(8));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(7));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("!08"));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("02"));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("03"));
|
||||
assertThat(topDocs.scoreDocs[3].doc, equalTo(3));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("04"));
|
||||
assertThat(topDocs.scoreDocs[4].doc, equalTo(4));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("06"));
|
||||
assertThat(topDocs.scoreDocs[5].doc, equalTo(6));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("08"));
|
||||
assertThat(topDocs.scoreDocs[6].doc, equalTo(1));
|
||||
assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(BytesRefFieldComparatorSource.MAX_TERM));
|
||||
assertThat(topDocs.scoreDocs[7].doc, equalTo(5));
|
||||
assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(BytesRefFieldComparatorSource.MAX_TERM));
|
||||
|
||||
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
|
||||
new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true)));
|
||||
assertThat(topDocs.totalHits, equalTo(8));
|
||||
assertThat(topDocs.scoreDocs.length, equalTo(8));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(6));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("10"));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(4));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("08"));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(3));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("06"));
|
||||
assertThat(topDocs.scoreDocs[3].doc, equalTo(0));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("04"));
|
||||
assertThat(topDocs.scoreDocs[4].doc, equalTo(2));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("03"));
|
||||
assertThat(topDocs.scoreDocs[5].doc, equalTo(7));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("!10"));
|
||||
assertThat(topDocs.scoreDocs[6].doc, equalTo(1));
|
||||
assertThat(((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(null));
|
||||
assertThat(topDocs.scoreDocs[7].doc, equalTo(5));
|
||||
assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null));
|
||||
}
|
||||
|
||||
protected abstract void fillExtendedMvSet() throws Exception;
|
||||
|
||||
}
|
||||
@ -33,8 +33,7 @@ import static org.hamcrest.Matchers.equalTo;
|
||||
|
||||
/**
|
||||
*/
|
||||
@Ignore("abstract")
|
||||
public abstract class NumericFieldDataTests extends AbstractStringFieldDataTests {
|
||||
public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImplTests {
|
||||
|
||||
protected abstract FieldDataType getFieldDataType();
|
||||
|
||||
@ -19,49 +19,37 @@
|
||||
|
||||
package org.elasticsearch.test.unit.index.fielddata;
|
||||
|
||||
import com.carrotsearch.randomizedtesting.annotations.Repeat;
|
||||
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.Field.Store;
|
||||
import org.apache.lucene.document.StringField;
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.search.join.ScoreMode;
|
||||
import org.apache.lucene.search.join.ToParentBlockJoinQuery;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.common.lucene.HashedBytesRef;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.BytesValues;
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
import org.apache.lucene.util.UnicodeUtil;
|
||||
import org.apache.lucene.util._TestUtil;
|
||||
import org.elasticsearch.common.lucene.search.NotFilter;
|
||||
import org.elasticsearch.common.lucene.search.TermFilter;
|
||||
import org.elasticsearch.common.lucene.search.XFilteredQuery;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.SortMode;
|
||||
import org.junit.Test;
|
||||
import org.elasticsearch.index.search.nested.NestedFieldComparatorSource;
|
||||
|
||||
import static org.hamcrest.Matchers.*;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
*/
|
||||
public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTests {
|
||||
|
||||
protected String one() {
|
||||
return "1";
|
||||
}
|
||||
|
||||
protected String two() {
|
||||
return "2";
|
||||
}
|
||||
|
||||
protected String three() {
|
||||
return "3";
|
||||
}
|
||||
|
||||
protected String four() {
|
||||
return "4";
|
||||
}
|
||||
|
||||
private String toString(Object value) {
|
||||
if (value instanceof BytesRef) {
|
||||
return ((BytesRef) value).utf8ToString();
|
||||
}
|
||||
return value.toString();
|
||||
}
|
||||
public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImplTests {
|
||||
|
||||
protected void fillSingleValueAllSet() throws Exception {
|
||||
Document d = new Document();
|
||||
@ -96,100 +84,6 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
|
||||
writer.deleteDocuments(new Term("_id", "1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeletedDocs() throws Exception {
|
||||
add2SingleValuedDocumentsAndDeleteOneOfThem();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicReaderContext readerContext = refreshReader();
|
||||
AtomicFieldData fieldData = indexFieldData.load(readerContext);
|
||||
assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l));
|
||||
BytesValues values = fieldData.getBytesValues();
|
||||
assertThat(fieldData.getNumDocs(), equalTo(2));
|
||||
assertThat(fieldData.getNumberUniqueValues(), equalTo(2l));
|
||||
for (int i = 0; i < fieldData.getNumDocs(); ++i) {
|
||||
assertThat(values.hasValue(i), equalTo(true));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingleValueAllSet() throws Exception {
|
||||
fillSingleValueAllSet();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicReaderContext readerContext = refreshReader();
|
||||
AtomicFieldData fieldData = indexFieldData.load(readerContext);
|
||||
assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
|
||||
assertThat(fieldData.getNumberUniqueValues(), equalTo(3l));
|
||||
assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l));
|
||||
|
||||
assertThat(fieldData.getNumDocs(), equalTo(3));
|
||||
|
||||
BytesValues bytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(bytesValues.isMultiValued(), equalTo(false));
|
||||
|
||||
assertThat(bytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(1), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
|
||||
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
|
||||
|
||||
BytesRef bytesRef = new BytesRef();
|
||||
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(one())));
|
||||
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(three())));
|
||||
|
||||
|
||||
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
BytesValues hashedBytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(1), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(one())));
|
||||
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
|
||||
|
||||
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
IndexSearcher searcher = new IndexSearcher(readerContext.reader());
|
||||
TopFieldDocs topDocs;
|
||||
|
||||
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
|
||||
new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN))));
|
||||
assertThat(topDocs.totalHits, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
|
||||
assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one()));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
|
||||
assertThat(toString(((FieldDoc) topDocs.scoreDocs[1]).fields[0]), equalTo(two()));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
|
||||
assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three()));
|
||||
|
||||
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
|
||||
new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true)));
|
||||
assertThat(topDocs.totalHits, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
|
||||
}
|
||||
|
||||
private HashedBytesRef convert(BytesValues values, int doc) {
|
||||
BytesRef ref = new BytesRef();
|
||||
return new HashedBytesRef(ref, values.getValueHashed(doc, ref));
|
||||
}
|
||||
|
||||
protected void fillSingleValueWithMissing() throws Exception {
|
||||
Document d = new Document();
|
||||
d.add(new StringField("_id", "1", Field.Store.NO));
|
||||
@ -207,68 +101,6 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
|
||||
writer.addDocument(d);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingleValueWithMissing() throws Exception {
|
||||
fillSingleValueWithMissing();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
|
||||
assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
|
||||
|
||||
assertThat(fieldData.getNumberUniqueValues(), equalTo(2l));
|
||||
assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l));
|
||||
assertThat(fieldData.getNumDocs(), equalTo(3));
|
||||
|
||||
BytesValues bytesValues = fieldData
|
||||
.getBytesValues();
|
||||
|
||||
assertThat(bytesValues.isMultiValued(), equalTo(false));
|
||||
|
||||
assertThat(bytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(bytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValue(1), nullValue());
|
||||
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
|
||||
|
||||
BytesRef bytesRef = new BytesRef();
|
||||
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
|
||||
assertThat(bytesRef, equalTo(new BytesRef()));
|
||||
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(three())));
|
||||
|
||||
|
||||
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
bytesValuesIter = bytesValues.getIter(1);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
BytesValues hashedBytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(new BytesRef())));
|
||||
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
|
||||
|
||||
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
hashedBytesValuesIter = hashedBytesValues.getIter(1);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
// TODO properly support missing....
|
||||
}
|
||||
|
||||
protected void fillMultiValueAllSet() throws Exception {
|
||||
Document d = new Document();
|
||||
d.add(new StringField("_id", "1", Field.Store.NO));
|
||||
@ -288,78 +120,6 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
|
||||
writer.addDocument(d);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiValueAllSet() throws Exception {
|
||||
fillMultiValueAllSet();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
|
||||
assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
|
||||
|
||||
assertThat(fieldData.getNumDocs(), equalTo(3));
|
||||
assertThat(fieldData.getNumberUniqueValues(), equalTo(4l));
|
||||
assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l));
|
||||
|
||||
BytesValues bytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(bytesValues.isMultiValued(), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(1), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
|
||||
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
|
||||
|
||||
BytesRef bytesRef = new BytesRef();
|
||||
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(one())));
|
||||
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(three())));
|
||||
|
||||
|
||||
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
BytesValues hashedBytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(1), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(one())));
|
||||
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
|
||||
|
||||
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(four())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
|
||||
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN))));
|
||||
assertThat(topDocs.totalHits, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs.length, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
|
||||
|
||||
topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true)));
|
||||
assertThat(topDocs.totalHits, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs.length, equalTo(3));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
|
||||
}
|
||||
|
||||
protected void fillMultiValueWithMissing() throws Exception {
|
||||
Document d = new Document();
|
||||
d.add(new StringField("_id", "1", Field.Store.NO));
|
||||
@ -378,129 +138,6 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
|
||||
writer.addDocument(d);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiValueWithMissing() throws Exception {
|
||||
fillMultiValueWithMissing();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
|
||||
assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l));
|
||||
|
||||
assertThat(fieldData.getNumDocs(), equalTo(3));
|
||||
assertThat(fieldData.getNumberUniqueValues(), equalTo(3l));
|
||||
assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l));
|
||||
|
||||
BytesValues bytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(bytesValues.isMultiValued(), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(bytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(bytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValue(1), nullValue());
|
||||
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
|
||||
|
||||
BytesRef bytesRef = new BytesRef();
|
||||
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
|
||||
assertThat(bytesRef, equalTo(new BytesRef()));
|
||||
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
|
||||
assertThat(bytesRef, equalTo(new BytesRef(three())));
|
||||
|
||||
|
||||
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
bytesValuesIter = bytesValues.getIter(1);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
BytesValues hashedBytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
|
||||
assertThat(hashedBytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
|
||||
|
||||
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(new BytesRef())));
|
||||
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
|
||||
|
||||
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
|
||||
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(four())));
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
hashedBytesValuesIter = hashedBytesValues.getIter(1);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
}
|
||||
|
||||
public void testMissingValueForAll() throws Exception {
|
||||
fillAllMissing();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
|
||||
// Some impls (FST) return size 0 and some (PagedBytes) do take size in the case no actual data is loaded
|
||||
assertThat(fieldData.getMemorySizeInBytes(), greaterThanOrEqualTo(0l));
|
||||
|
||||
assertThat(fieldData.getNumDocs(), equalTo(3));
|
||||
assertThat(fieldData.getNumberUniqueValues(), equalTo(0l));
|
||||
assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), equalTo(0l));
|
||||
|
||||
BytesValues bytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(bytesValues.isMultiValued(), equalTo(false));
|
||||
|
||||
assertThat(bytesValues.hasValue(0), equalTo(false));
|
||||
assertThat(bytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(bytesValues.hasValue(2), equalTo(false));
|
||||
|
||||
assertThat(bytesValues.getValue(0), nullValue());
|
||||
assertThat(bytesValues.getValue(1), nullValue());
|
||||
assertThat(bytesValues.getValue(2), nullValue());
|
||||
|
||||
BytesRef bytesRef = new BytesRef();
|
||||
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef()));
|
||||
assertThat(bytesRef, equalTo(new BytesRef()));
|
||||
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
|
||||
assertThat(bytesRef, equalTo(new BytesRef()));
|
||||
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef()));
|
||||
assertThat(bytesRef, equalTo(new BytesRef()));
|
||||
|
||||
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
bytesValuesIter = bytesValues.getIter(1);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
bytesValuesIter = bytesValues.getIter(2);
|
||||
assertThat(bytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
BytesValues hashedBytesValues = fieldData.getBytesValues();
|
||||
|
||||
assertThat(hashedBytesValues.hasValue(0), equalTo(false));
|
||||
assertThat(hashedBytesValues.hasValue(1), equalTo(false));
|
||||
assertThat(hashedBytesValues.hasValue(2), equalTo(false));
|
||||
|
||||
assertThat(hashedBytesValues.getValue(0), nullValue());
|
||||
assertThat(hashedBytesValues.getValue(1), nullValue());
|
||||
assertThat(hashedBytesValues.getValue(2), nullValue());
|
||||
|
||||
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
hashedBytesValuesIter = hashedBytesValues.getIter(1);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
|
||||
hashedBytesValuesIter = hashedBytesValues.getIter(2);
|
||||
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
|
||||
}
|
||||
|
||||
protected void fillAllMissing() throws Exception {
|
||||
Document d = new Document();
|
||||
d.add(new StringField("_id", "1", Field.Store.NO));
|
||||
@ -515,55 +152,6 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
|
||||
writer.addDocument(d);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSortMultiValuesFields() throws Exception {
|
||||
fillExtendedMvSet();
|
||||
IndexFieldData indexFieldData = getForField("value");
|
||||
|
||||
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
|
||||
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10,
|
||||
new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN))));
|
||||
assertThat(topDocs.totalHits, equalTo(8));
|
||||
assertThat(topDocs.scoreDocs.length, equalTo(8));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
|
||||
assertThat(((FieldDoc) topDocs.scoreDocs[0]).fields[0], equalTo(null));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(5));
|
||||
assertThat(((FieldDoc) topDocs.scoreDocs[1]).fields[0], equalTo(null));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(7));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("!08"));
|
||||
assertThat(topDocs.scoreDocs[3].doc, equalTo(0));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("02"));
|
||||
assertThat(topDocs.scoreDocs[4].doc, equalTo(2));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("03"));
|
||||
assertThat(topDocs.scoreDocs[5].doc, equalTo(3));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("04"));
|
||||
assertThat(topDocs.scoreDocs[6].doc, equalTo(4));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("06"));
|
||||
assertThat(topDocs.scoreDocs[7].doc, equalTo(6));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]).utf8ToString(), equalTo("08"));
|
||||
|
||||
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
|
||||
new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true)));
|
||||
assertThat(topDocs.totalHits, equalTo(8));
|
||||
assertThat(topDocs.scoreDocs.length, equalTo(8));
|
||||
assertThat(topDocs.scoreDocs[0].doc, equalTo(6));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("10"));
|
||||
assertThat(topDocs.scoreDocs[1].doc, equalTo(4));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("08"));
|
||||
assertThat(topDocs.scoreDocs[2].doc, equalTo(3));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("06"));
|
||||
assertThat(topDocs.scoreDocs[3].doc, equalTo(0));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("04"));
|
||||
assertThat(topDocs.scoreDocs[4].doc, equalTo(2));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("03"));
|
||||
assertThat(topDocs.scoreDocs[5].doc, equalTo(7));
|
||||
assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("!10"));
|
||||
assertThat(topDocs.scoreDocs[6].doc, equalTo(1));
|
||||
assertThat(((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(null));
|
||||
assertThat(topDocs.scoreDocs[7].doc, equalTo(5));
|
||||
assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null));
|
||||
}
|
||||
|
||||
protected void fillExtendedMvSet() throws Exception {
|
||||
Document d = new Document();
|
||||
d.add(new StringField("_id", "1", Field.Store.NO));
|
||||
@ -615,4 +203,224 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest
|
||||
writer.addDocument(d);
|
||||
}
|
||||
|
||||
@Repeat(iterations=10)
|
||||
public void testActualMissingValue() throws IOException {
|
||||
testActualMissingValue(false);
|
||||
}
|
||||
|
||||
@Repeat(iterations=10)
|
||||
public void testActualMissingValueReverse() throws IOException {
|
||||
testActualMissingValue(true);
|
||||
}
|
||||
|
||||
public void testActualMissingValue(boolean reverse) throws IOException {
|
||||
// missing value is set to an actual value
|
||||
Document d = new Document();
|
||||
final StringField s = new StringField("value", "", Field.Store.YES);
|
||||
d.add(s);
|
||||
final String[] values = new String[randomIntBetween(2, 30)];
|
||||
for (int i = 1; i < values.length; ++i) {
|
||||
values[i] = _TestUtil.randomUnicodeString(getRandom());
|
||||
}
|
||||
final int numDocs = atLeast(100);
|
||||
for (int i = 0; i < numDocs; ++i) {
|
||||
final String value = RandomPicks.randomFrom(getRandom(), values);
|
||||
if (value == null) {
|
||||
writer.addDocument(new Document());
|
||||
} else {
|
||||
s.setStringValue(value);
|
||||
writer.addDocument(d);
|
||||
}
|
||||
if (randomInt(10) == 0) {
|
||||
writer.commit();
|
||||
}
|
||||
}
|
||||
|
||||
final IndexFieldData indexFieldData = getForField("value");
|
||||
final String missingValue = values[1];
|
||||
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
|
||||
XFieldComparatorSource comparator = indexFieldData.comparatorSource(missingValue, SortMode.MIN);
|
||||
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse)));
|
||||
assertEquals(numDocs, topDocs.totalHits);
|
||||
BytesRef previousValue = reverse ? UnicodeUtil.BIG_TERM : new BytesRef();
|
||||
for (int i = 0; i < topDocs.scoreDocs.length; ++i) {
|
||||
final String docValue = searcher.doc(topDocs.scoreDocs[i].doc).get("value");
|
||||
final BytesRef value = new BytesRef(docValue == null ? missingValue : docValue);
|
||||
if (reverse) {
|
||||
assertTrue(previousValue.compareTo(value) >= 0);
|
||||
} else {
|
||||
assertTrue(previousValue.compareTo(value) <= 0);
|
||||
}
|
||||
previousValue = value;
|
||||
}
|
||||
searcher.getIndexReader().close();
|
||||
}
|
||||
|
||||
@Repeat(iterations=3)
|
||||
public void testSortMissingFirst() throws IOException {
|
||||
testSortMissing(true, false);
|
||||
}
|
||||
|
||||
@Repeat(iterations=3)
|
||||
public void testSortMissingFirstReverse() throws IOException {
|
||||
testSortMissing(true, true);
|
||||
}
|
||||
|
||||
@Repeat(iterations=3)
|
||||
public void testSortMissingLast() throws IOException {
|
||||
testSortMissing(false, false);
|
||||
}
|
||||
|
||||
@Repeat(iterations=3)
|
||||
public void testSortMissingLastReverse() throws IOException {
|
||||
testSortMissing(false, true);
|
||||
}
|
||||
|
||||
public void testSortMissing(boolean first, boolean reverse) throws IOException {
|
||||
Document d = new Document();
|
||||
final StringField s = new StringField("value", "", Field.Store.YES);
|
||||
d.add(s);
|
||||
final String[] values = new String[randomIntBetween(2, 10)];
|
||||
for (int i = 1; i < values.length; ++i) {
|
||||
values[i] = _TestUtil.randomUnicodeString(getRandom());
|
||||
}
|
||||
final int numDocs = atLeast(100);
|
||||
for (int i = 0; i < numDocs; ++i) {
|
||||
final String value = RandomPicks.randomFrom(getRandom(), values);
|
||||
if (value == null) {
|
||||
writer.addDocument(new Document());
|
||||
} else {
|
||||
s.setStringValue(value);
|
||||
writer.addDocument(d);
|
||||
}
|
||||
if (randomInt(10) == 0) {
|
||||
writer.commit();
|
||||
}
|
||||
}
|
||||
final IndexFieldData indexFieldData = getForField("value");
|
||||
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
|
||||
XFieldComparatorSource comparator = indexFieldData.comparatorSource(first ? "_first" : "_last", SortMode.MIN);
|
||||
TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse)));
|
||||
assertEquals(numDocs, topDocs.totalHits);
|
||||
BytesRef previousValue = first ? null : reverse ? UnicodeUtil.BIG_TERM : new BytesRef();
|
||||
for (int i = 0; i < topDocs.scoreDocs.length; ++i) {
|
||||
final String docValue = searcher.doc(topDocs.scoreDocs[i].doc).get("value");
|
||||
if (first && docValue == null) {
|
||||
assertNull(previousValue);
|
||||
} else if (!first && docValue != null) {
|
||||
assertNotNull(previousValue);
|
||||
}
|
||||
final BytesRef value = docValue == null ? null : new BytesRef(docValue);
|
||||
if (previousValue != null && value != null) {
|
||||
if (reverse) {
|
||||
assertTrue(previousValue.compareTo(value) >= 0);
|
||||
} else {
|
||||
assertTrue(previousValue.compareTo(value) <= 0);
|
||||
}
|
||||
}
|
||||
previousValue = value;
|
||||
}
|
||||
searcher.getIndexReader().close();
|
||||
}
|
||||
|
||||
@Repeat(iterations=3)
|
||||
public void testNestedSortingMin() throws IOException {
|
||||
testNestedSorting(SortMode.MIN);
|
||||
}
|
||||
|
||||
@Repeat(iterations=3)
|
||||
public void testNestedSortingMax() throws IOException {
|
||||
testNestedSorting(SortMode.MAX);
|
||||
}
|
||||
|
||||
public void testNestedSorting(SortMode sortMode) throws IOException {
|
||||
final String[] values = new String[randomIntBetween(2, 20)];
|
||||
for (int i = 0; i < values.length; ++i) {
|
||||
values[i] = _TestUtil.randomSimpleString(getRandom());
|
||||
}
|
||||
final int numParents = atLeast(100);
|
||||
List<Document> docs = new ArrayList<Document>();
|
||||
final OpenBitSet parents = new OpenBitSet();
|
||||
for (int i = 0; i < numParents; ++i) {
|
||||
docs.clear();
|
||||
final int numChildren = randomInt(4);
|
||||
for (int j = 0; j < numChildren; ++j) {
|
||||
final Document child = new Document();
|
||||
final int numValues = randomInt(3);
|
||||
for (int k = 0; k < numValues; ++k) {
|
||||
final String value = RandomPicks.randomFrom(getRandom(), values);
|
||||
child.add(new StringField("text", value, Store.YES));
|
||||
}
|
||||
docs.add(child);
|
||||
}
|
||||
final Document parent = new Document();
|
||||
parent.add(new StringField("type", "parent", Store.YES));
|
||||
final String value = RandomPicks.randomFrom(getRandom(), values);
|
||||
if (value != null) {
|
||||
parent.add(new StringField("text", value, Store.YES));
|
||||
}
|
||||
docs.add(parent);
|
||||
parents.set(parents.prevSetBit(parents.length() - 1) + docs.size());
|
||||
writer.addDocuments(docs);
|
||||
if (randomInt(10) == 0) {
|
||||
writer.commit();
|
||||
}
|
||||
}
|
||||
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
|
||||
IndexFieldData<?> fieldData = getForField("text");
|
||||
final BytesRef missingValue;
|
||||
switch (randomInt(4)) {
|
||||
case 0:
|
||||
missingValue = new BytesRef();
|
||||
break;
|
||||
case 1:
|
||||
missingValue = BytesRefFieldComparatorSource.MAX_TERM;
|
||||
break;
|
||||
case 2:
|
||||
missingValue = new BytesRef(RandomPicks.randomFrom(getRandom(), values));
|
||||
break;
|
||||
default:
|
||||
missingValue = new BytesRef(_TestUtil.randomSimpleString(getRandom()));
|
||||
break;
|
||||
}
|
||||
BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode);
|
||||
Filter parentFilter = new TermFilter(new Term("type", "parent"));
|
||||
Filter childFilter = new NotFilter(parentFilter);
|
||||
NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
|
||||
ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new CachingWrapperFilter(parentFilter), ScoreMode.None);
|
||||
Sort sort = new Sort(new SortField("text", nestedComparatorSource));
|
||||
TopFieldDocs topDocs = searcher.search(query, randomIntBetween(1, numParents), sort);
|
||||
assertTrue(topDocs.scoreDocs.length > 0);
|
||||
BytesRef previous = null;
|
||||
for (int i = 0; i < topDocs.scoreDocs.length; ++i) {
|
||||
final int docID = topDocs.scoreDocs[i].doc;
|
||||
assert parents.get(docID);
|
||||
BytesRef cmpValue = null;
|
||||
for (int child = parents.prevSetBit(docID - 1) + 1; child < docID; ++child) {
|
||||
String[] vals = searcher.doc(child).getValues("text");
|
||||
if (vals.length == 0) {
|
||||
vals = new String[] {missingValue.utf8ToString()};
|
||||
}
|
||||
for (String value : vals) {
|
||||
final BytesRef bytesValue = new BytesRef(value);
|
||||
if (cmpValue == null) {
|
||||
cmpValue = bytesValue;
|
||||
} else if (sortMode == SortMode.MIN && bytesValue.compareTo(cmpValue) < 0) {
|
||||
cmpValue = bytesValue;
|
||||
} else if (sortMode == SortMode.MAX && bytesValue.compareTo(cmpValue) > 0) {
|
||||
cmpValue = bytesValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (cmpValue == null) {
|
||||
cmpValue = missingValue;
|
||||
}
|
||||
if (previous != null) {
|
||||
assertNotNull(cmpValue);
|
||||
assertTrue(previous.utf8ToString() + " / " + cmpValue.utf8ToString(), previous.compareTo(cmpValue) <= 0);
|
||||
}
|
||||
previous = cmpValue;
|
||||
}
|
||||
searcher.getIndexReader().close();
|
||||
}
|
||||
}
|
||||
|
||||
@ -28,7 +28,7 @@ import org.elasticsearch.index.fielddata.FieldDataType;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class DoubleFieldDataTests extends NumericFieldDataTests {
|
||||
public class DoubleFieldDataTests extends AbstractNumericFieldDataTests {
|
||||
|
||||
@Override
|
||||
protected FieldDataType getFieldDataType() {
|
||||
|
||||
@ -28,7 +28,7 @@ import org.elasticsearch.index.fielddata.FieldDataType;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class FloatFieldDataTests extends NumericFieldDataTests {
|
||||
public class FloatFieldDataTests extends AbstractNumericFieldDataTests {
|
||||
|
||||
@Override
|
||||
protected FieldDataType getFieldDataType() {
|
||||
|
||||
@ -46,7 +46,7 @@ import static org.hamcrest.Matchers.instanceOf;
|
||||
/**
|
||||
* Tests for all integer types (byte, short, int, long).
|
||||
*/
|
||||
public class LongFieldDataTests extends NumericFieldDataTests {
|
||||
public class LongFieldDataTests extends AbstractNumericFieldDataTests {
|
||||
|
||||
@Override
|
||||
protected FieldDataType getFieldDataType() {
|
||||
|
||||
@ -0,0 +1,90 @@
|
||||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.test.unit.index.fielddata;
|
||||
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.ScriptDocValues;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.SortMode;
|
||||
import org.elasticsearch.index.mapper.FieldMapper.Names;
|
||||
|
||||
/** Returns an implementation based on paged bytes which doesn't implement WithOrdinals in order to visit different paths in the code,
|
||||
* eg. BytesRefFieldComparatorSource makes decisions based on whether the field data implements WithOrdinals. */
|
||||
public class NoOrdinalsStringFieldDataTests extends PagedBytesStringFieldDataTests {
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public IndexFieldData<AtomicFieldData<ScriptDocValues>> getForField(String fieldName) {
|
||||
final IndexFieldData<?> in = super.getForField(fieldName);
|
||||
return new IndexFieldData<AtomicFieldData<ScriptDocValues>>() {
|
||||
|
||||
@Override
|
||||
public Index index() {
|
||||
return in.index();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Names getFieldNames() {
|
||||
return in.getFieldNames();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean valuesOrdered() {
|
||||
return in.valuesOrdered();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AtomicFieldData<ScriptDocValues> load(AtomicReaderContext context) {
|
||||
return in.load(context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AtomicFieldData<ScriptDocValues> loadDirect(AtomicReaderContext context) throws Exception {
|
||||
return in.loadDirect(context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public XFieldComparatorSource comparatorSource(Object missingValue, SortMode sortMode) {
|
||||
return new BytesRefFieldComparatorSource(this, missingValue, sortMode);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
in.clear();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear(IndexReader reader) {
|
||||
in.clear(reader);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getHighestNumberOfSeenUniqueValues() {
|
||||
return in.getHighestNumberOfSeenUniqueValues();
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
@ -46,7 +46,6 @@ import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
|
||||
/**
|
||||
@ -216,7 +215,7 @@ public class NestedSortingTests extends AbstractFieldDataTests {
|
||||
SortMode sortMode = SortMode.MIN;
|
||||
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false));
|
||||
PagedBytesIndexFieldData indexFieldData = getForField("field2");
|
||||
BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(indexFieldData, sortMode);
|
||||
BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode);
|
||||
Filter parentFilter = new TermFilter(new Term("__type", "parent"));
|
||||
Filter childFilter = new NotFilter(parentFilter);
|
||||
NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user