diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 32f9f5bc1cb..1c2bc8bd796 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -21,6 +21,9 @@ package org.elasticsearch.index.fielddata.fieldcomparator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.index.fielddata.IndexFieldData; import java.io.IOException; @@ -29,12 +32,23 @@ import java.io.IOException; */ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparatorSource { + /** UTF-8 term containing a single code point: {@link Character#MAX_CODE_POINT} which will compare greater than all other index terms + * since {@link Character#MAX_CODE_POINT} is a noncharacter and thus shouldn't appear in an index term. */ + public static final BytesRef MAX_TERM; + static { + MAX_TERM = new BytesRef(); + final char[] chars = Character.toChars(Character.MAX_CODE_POINT); + UnicodeUtil.UTF16toUTF8(chars, 0, chars.length, MAX_TERM); + } + private final IndexFieldData indexFieldData; private final SortMode sortMode; + private final Object missingValue; - public BytesRefFieldComparatorSource(IndexFieldData indexFieldData, SortMode sortMode) { + public BytesRefFieldComparatorSource(IndexFieldData indexFieldData, Object missingValue, SortMode sortMode) { this.indexFieldData = indexFieldData; this.sortMode = sortMode; + this.missingValue = missingValue; } @Override @@ -45,9 +59,25 @@ public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparat @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { assert fieldname.equals(indexFieldData.getFieldNames().indexName()); - if (indexFieldData.valuesOrdered() && indexFieldData instanceof IndexFieldData.WithOrdinals) { - return new BytesRefOrdValComparator((IndexFieldData.WithOrdinals) indexFieldData, numHits, sortMode); + BytesRef missingBytes = null; + if (missingValue == null || "_last".equals(missingValue)) { + missingBytes = reversed ? null : MAX_TERM; + } else if ("_first".equals(missingValue)) { + missingBytes = reversed ? MAX_TERM : null; + } else if (missingValue instanceof BytesRef) { + missingBytes = (BytesRef) missingValue; + } else if (missingValue instanceof String) { + missingBytes = new BytesRef((String) missingValue); + } else if (missingValue instanceof byte[]) { + missingBytes = new BytesRef((byte[]) missingValue); + } else { + throw new ElasticSearchIllegalArgumentException("Unsupported missing value: " + missingValue); } - return new BytesRefValComparator(indexFieldData, numHits, sortMode); + + if (indexFieldData.valuesOrdered() && indexFieldData instanceof IndexFieldData.WithOrdinals) { + return new BytesRefOrdValComparator((IndexFieldData.WithOrdinals) indexFieldData, numHits, sortMode, missingBytes); + } + return new BytesRefValComparator(indexFieldData, numHits, sortMode, missingBytes); } + } diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java index 9c87e16e00d..2a9bbbb715e 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java +++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java @@ -38,12 +38,19 @@ import java.io.IOException; * to large results, this comparator will be much faster * than {@link org.apache.lucene.search.FieldComparator.TermValComparator}. For very small * result sets it may be slower. + * + * Internally this comparator multiplies ordinals by 4 so that virtual ordinals can be inserted in-between the original field data ordinals. + * Thanks to this, an ordinal for the missing value and the bottom value can be computed and all ordinals are directly comparable. For example, + * if the field data ordinals are (a,1), (b,2) and (c,3), they will be internally stored as (a,4), (b,8), (c,12). Then the ordinal for the + * missing value will be computed by binary searching. For example, if the missing value is 'ab', it will be assigned 6 as an ordinal (between + * 'a' and 'b'. And if the bottom value is 'ac', it will be assigned 7 as an ordinal (between 'ab' and 'b'). */ -public final class BytesRefOrdValComparator extends FieldComparator { +public final class BytesRefOrdValComparator extends NestedWrappableComparator { final IndexFieldData.WithOrdinals indexFieldData; + final BytesRef missingValue; - /* Ords for each slot. + /* Ords for each slot, times 4. @lucene.internal */ final long[] ords; @@ -67,6 +74,7 @@ public final class BytesRefOrdValComparator extends FieldComparator { /* Current reader's doc ord/values. @lucene.internal */ BytesValues.WithOrdinals termsIndex; + long missingOrd; /* Bottom slot, or -1 if queue isn't full yet @lucene.internal */ @@ -77,21 +85,12 @@ public final class BytesRefOrdValComparator extends FieldComparator { @lucene.internal */ long bottomOrd; - /* True if current bottom slot matches the current - reader. - @lucene.internal */ - boolean bottomSameReader; - - /* Bottom value (same as values[bottomSlot] once - bottomSlot is set). Cached for faster compares. - @lucene.internal */ - BytesRef bottomValue; - final BytesRef tempBR = new BytesRef(); - public BytesRefOrdValComparator(IndexFieldData.WithOrdinals indexFieldData, int numHits, SortMode sortMode) { + public BytesRefOrdValComparator(IndexFieldData.WithOrdinals indexFieldData, int numHits, SortMode sortMode, BytesRef missingValue) { this.indexFieldData = indexFieldData; this.sortMode = sortMode; + this.missingValue = missingValue; ords = new long[numHits]; values = new BytesRef[numHits]; readerGen = new int[numHits]; @@ -121,34 +120,37 @@ public final class BytesRefOrdValComparator extends FieldComparator { throw new UnsupportedOperationException(); } + @Override + public int compareBottomMissing() { + throw new UnsupportedOperationException(); + } + @Override public void copy(int slot, int doc) { throw new UnsupportedOperationException(); } @Override - public int compareDocToValue(int doc, BytesRef value) { - BytesRef docValue = termsIndex.getValue(doc); - if (docValue == null) { - if (value == null) { - return 0; - } - return -1; - } else if (value == null) { - return 1; - } - return docValue.compareTo(value); + public void missing(int slot) { + throw new UnsupportedOperationException(); } - /** - * Base class for specialized (per bit width of the - * ords) per-segment comparator. NOTE: this is messy; - * we do this only because hotspot can't reliably inline - * the underlying array access when looking up doc->ord - * - * @lucene.internal - */ - abstract class PerSegmentComparator extends FieldComparator { + @Override + public int compareDocToValue(int doc, BytesRef value) { + throw new UnsupportedOperationException(); + } + + class PerSegmentComparator extends NestedWrappableComparator { + final Ordinals.Docs readerOrds; + final BytesValues.WithOrdinals termsIndex; + + public PerSegmentComparator(BytesValues.WithOrdinals termsIndex) { + this.readerOrds = termsIndex.ordinals(); + this.termsIndex = termsIndex; + if (readerOrds.getNumOrds() > Long.MAX_VALUE / 4) { + throw new IllegalStateException("Current terms index pretends it has more than " + (Long.MAX_VALUE / 4) + " ordinals, which is unsupported by this impl"); + } + } @Override public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { @@ -185,225 +187,109 @@ public final class BytesRefOrdValComparator extends FieldComparator { @Override public int compareDocToValue(int doc, BytesRef value) { - return BytesRefOrdValComparator.this.compareDocToValue(doc, value); + final long ord = getOrd(doc); + final BytesRef docValue = ord == 0 ? missingValue : termsIndex.getValueByOrd(ord); + return compareValues(docValue, value); } - } - // Used per-segment when bit width of doc->ord is 8: - private final class ByteOrdComparator extends PerSegmentComparator { - private final byte[] readerOrds; - private final BytesValues.WithOrdinals termsIndex; - private final int docBase; - - public ByteOrdComparator(byte[] readerOrds, BytesValues.WithOrdinals termsIndex, int docBase) { - this.readerOrds = readerOrds; - this.termsIndex = termsIndex; - this.docBase = docBase; + protected long getOrd(int doc) { + return readerOrds.getOrd(doc); } @Override public int compareBottom(int doc) { assert bottomSlot != -1; - final int docOrd = (readerOrds[doc] & 0xFF); - if (bottomSameReader) { - // ord is precisely comparable, even in the equal case - return (int) bottomOrd - docOrd; - } else if (bottomOrd >= docOrd) { - // the equals case always means bottom is > doc - // (because we set bottomOrd to the lower bound in - // setBottom): - return 1; - } else { - return -1; - } + final long docOrd = getOrd(doc); + final long comparableOrd = docOrd == 0 ? missingOrd : docOrd << 2; + return LongValuesComparator.compare(bottomOrd, comparableOrd); + } + + @Override + public int compareBottomMissing() { + assert bottomSlot != -1; + return LongValuesComparator.compare(bottomOrd, missingOrd); } @Override public void copy(int slot, int doc) { - final int ord = readerOrds[doc] & 0xFF; - ords[slot] = ord; + final long ord = getOrd(doc); if (ord == 0) { - values[slot] = null; + ords[slot] = missingOrd; + values[slot] = missingValue; } else { assert ord > 0; - if (values[slot] == null) { + ords[slot] = ord << 2; + if (values[slot] == null || values[slot] == missingValue) { values[slot] = new BytesRef(); } termsIndex.getValueScratchByOrd(ord, values[slot]); } readerGen[slot] = currentReaderGen; } - } - - // Used per-segment when bit width of doc->ord is 16: - private final class ShortOrdComparator extends PerSegmentComparator { - private final short[] readerOrds; - private final BytesValues.WithOrdinals termsIndex; - private final int docBase; - - public ShortOrdComparator(short[] readerOrds, BytesValues.WithOrdinals termsIndex, int docBase) { - this.readerOrds = readerOrds; - this.termsIndex = termsIndex; - this.docBase = docBase; - } @Override - public int compareBottom(int doc) { - assert bottomSlot != -1; - final int docOrd = (readerOrds[doc] & 0xFFFF); - if (bottomSameReader) { - // ord is precisely comparable, even in the equal case - return (int) bottomOrd - docOrd; - } else if (bottomOrd >= docOrd) { - // the equals case always means bottom is > doc - // (because we set bottomOrd to the lower bound in - // setBottom): - return 1; - } else { - return -1; - } - } - - @Override - public void copy(int slot, int doc) { - final int ord = readerOrds[doc] & 0xFFFF; - ords[slot] = ord; - if (ord == 0) { - values[slot] = null; - } else { - assert ord > 0; - if (values[slot] == null) { - values[slot] = new BytesRef(); - } - termsIndex.getValueScratchByOrd(ord, values[slot]); - } - readerGen[slot] = currentReaderGen; + public void missing(int slot) { + ords[slot] = missingOrd; + values[slot] = missingValue; } } - // Used per-segment when bit width of doc->ord is 32: - private final class IntOrdComparator extends PerSegmentComparator { - private final int[] readerOrds; - private final BytesValues.WithOrdinals termsIndex; - private final int docBase; - - public IntOrdComparator(int[] readerOrds, BytesValues.WithOrdinals termsIndex, int docBase) { - this.readerOrds = readerOrds; - this.termsIndex = termsIndex; - this.docBase = docBase; + // for assertions + private boolean consistentInsertedOrd(BytesValues.WithOrdinals termsIndex, long ord, BytesRef value) { + assert ord >= 0 : ord; + assert (ord == 0) == (value == null) : "ord=" + ord + ", value=" + value; + final long previousOrd = ord >>> 2; + final long nextOrd = previousOrd + 1; + final BytesRef previous = previousOrd == 0 ? null : termsIndex.getValueByOrd(previousOrd); + if ((ord & 3) == 0) { // there was an existing ord with the inserted value + assert compareValues(previous, value) == 0; + } else { + assert compareValues(previous, value) < 0; } - - @Override - public int compareBottom(int doc) { - assert bottomSlot != -1; - final int docOrd = readerOrds[doc]; - if (bottomSameReader) { - // ord is precisely comparable, even in the equal case - return (int) bottomOrd - docOrd; - } else if (bottomOrd >= docOrd) { - // the equals case always means bottom is > doc - // (because we set bottomOrd to the lower bound in - // setBottom): - return 1; - } else { - return -1; - } - } - - @Override - public void copy(int slot, int doc) { - final int ord = readerOrds[doc]; - ords[slot] = ord; - if (ord == 0) { - values[slot] = null; - } else { - assert ord > 0; - if (values[slot] == null) { - values[slot] = new BytesRef(); - } - termsIndex.getValueScratchByOrd(ord, values[slot]); - } - readerGen[slot] = currentReaderGen; + if (nextOrd < termsIndex.ordinals().getMaxOrd()) { + final BytesRef next = termsIndex.getValueByOrd(nextOrd); + assert compareValues(value, next) < 0; } + return true; } - // Used per-segment when bit width is not a native array - // size (8, 16, 32): - final class AnyOrdComparator extends PerSegmentComparator { - private final IndexFieldData fieldData; - private final Ordinals.Docs readerOrds; - private final BytesValues.WithOrdinals termsIndex; - private final int docBase; - - public AnyOrdComparator(IndexFieldData fieldData, BytesValues.WithOrdinals termsIndex, int docBase) { - this.fieldData = fieldData; - this.readerOrds = termsIndex.ordinals(); - this.termsIndex = termsIndex; - this.docBase = docBase; - } - - @Override - public int compareBottom(int doc) { - assert bottomSlot != -1; - final long docOrd = readerOrds.getOrd(doc); - if (bottomSameReader) { - // ord is precisely comparable, even in the equal case - return LongValuesComparator.compare(bottomOrd, docOrd); - } else if (bottomOrd >= docOrd) { - // the equals case always means bottom is > doc - // (because we set bottomOrd to the lower bound in - // setBottom): - return 1; - } else { - return -1; - } - } - - @Override - public void copy(int slot, int doc) { - final long ord = readerOrds.getOrd(doc); - ords[slot] = ord; - if (ord == 0) { - values[slot] = null; - } else { - assert ord > 0; - if (values[slot] == null) { - values[slot] = new BytesRef(); - } - termsIndex.getValueScratchByOrd(ord, values[slot]); - } - readerGen[slot] = currentReaderGen; + // find where to insert an ord in the current terms index + private long ordInCurrentReader(BytesValues.WithOrdinals termsIndex, BytesRef value) { + final long docOrd = binarySearch(termsIndex, value); + assert docOrd != -1; // would mean smaller than null + final long ord; + if (docOrd >= 0) { + // value exists in the current segment + ord = docOrd << 2; + } else { + // value doesn't exist, use the ord between the previous and the next term + ord = ((-2 - docOrd) << 2) + 2; } + assert (ord & 1) == 0; + return ord; } @Override public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - final int docBase = context.docBase; termsIndex = indexFieldData.load(context).getBytesValues(); + assert termsIndex.ordinals() != null && termsIndex.ordinals().ordinals() != null; + if (missingValue == null) { + missingOrd = 0; + } else { + missingOrd = ordInCurrentReader(termsIndex, missingValue); + assert consistentInsertedOrd(termsIndex, missingOrd, missingValue); + } FieldComparator perSegComp = null; assert termsIndex.ordinals() != null && termsIndex.ordinals().ordinals() != null; if (termsIndex.isMultiValued()) { - perSegComp = new MultiAnyOrdComparator(termsIndex); - } else { - final Ordinals.Docs docToOrd = termsIndex.ordinals(); - if (docToOrd.ordinals().hasSingleArrayBackingStorage()) { - Object ordsStorage = docToOrd.ordinals().getBackingStorage(); - if (ordsStorage instanceof byte[]) { - perSegComp = new ByteOrdComparator((byte[]) ordsStorage, termsIndex, docBase); - } else if (ordsStorage instanceof short[]) { - perSegComp = new ShortOrdComparator((short[]) ordsStorage, termsIndex, docBase); - } else if (ordsStorage instanceof int[]) { - perSegComp = new IntOrdComparator((int[]) ordsStorage, termsIndex, docBase); + perSegComp = new PerSegmentComparator(termsIndex) { + @Override + protected long getOrd(int doc) { + return getRelevantOrd(readerOrds, doc, sortMode); } - } - // Don't specialize the long[] case since it's not - // possible, ie, worse case is MAX_INT-1 docs with - // every one having a unique value. - - // TODO: ES - should we optimize for the PackedInts.Reader case as well? - if (perSegComp == null) { - perSegComp = new AnyOrdComparator(indexFieldData, termsIndex, docBase); - } + }; + } else { + perSegComp = new PerSegmentComparator(termsIndex); } currentReaderGen++; if (bottomSlot != -1) { @@ -415,32 +301,29 @@ public final class BytesRefOrdValComparator extends FieldComparator { @Override public void setBottom(final int bottom) { bottomSlot = bottom; + final BytesRef bottomValue = values[bottomSlot]; - bottomValue = values[bottomSlot]; - if (currentReaderGen == readerGen[bottomSlot]) { + if (bottomValue == null) { + bottomOrd = 0; + } else if (currentReaderGen == readerGen[bottomSlot]) { bottomOrd = ords[bottomSlot]; - bottomSameReader = true; } else { - if (bottomValue == null) { - // 0 ord is null for all segments - assert ords[bottomSlot] == 0; - bottomOrd = 0; - bottomSameReader = true; - readerGen[bottomSlot] = currentReaderGen; - } else { - final long index = binarySearch(termsIndex, bottomValue); - if (index < 0) { - bottomOrd = -index - 2; - bottomSameReader = false; - } else { - bottomOrd = index; - // exact value match - bottomSameReader = true; - readerGen[bottomSlot] = currentReaderGen; - ords[bottomSlot] = bottomOrd; + // insert an ord + bottomOrd = ordInCurrentReader(termsIndex, bottomValue); + if (bottomOrd == missingOrd) { + // bottomValue and missingValue and in-between the same field data values -> tie-break + // this is why we multiply ords by 4 + assert missingValue != null; + final int cmp = bottomValue.compareTo(missingValue); + if (cmp < 0) { + --bottomOrd; + } else if (cmp > 0) { + ++bottomOrd; } } + assert consistentInsertedOrd(termsIndex, bottomOrd, bottomValue); } + readerGen[bottomSlot] = currentReaderGen; } @Override @@ -454,7 +337,7 @@ public final class BytesRefOrdValComparator extends FieldComparator { final protected static long binarySearch(BytesValues.WithOrdinals a, BytesRef key, long low, long high) { assert a.getValueByOrd(high) == null | a.getValueByOrd(high) != null; // make sure we actually can get these values - assert a.getValueByOrd(low) == null | a.getValueByOrd(low) != null; + assert low == high + 1 || a.getValueByOrd(low) == null | a.getValueByOrd(low) != null; while (low <= high) { long mid = (low + high) >>> 1; BytesRef midVal = a.getValueByOrd(mid); @@ -475,65 +358,6 @@ public final class BytesRefOrdValComparator extends FieldComparator { return -(low + 1); } - - class MultiAnyOrdComparator extends PerSegmentComparator { - - private final BytesValues.WithOrdinals termsIndex; - private final Ordinals.Docs readerOrds; - - private MultiAnyOrdComparator(BytesValues.WithOrdinals termsIndex) { - this.termsIndex = termsIndex; - this.readerOrds = termsIndex.ordinals(); - } - - @Override - public int compareBottom(int doc) throws IOException { - final long docOrd = getRelevantOrd(readerOrds, doc, sortMode); - if (bottomSameReader) { - // ord is precisely comparable, even in the equal case - return LongValuesComparator.compare(bottomOrd, docOrd); - } else if (bottomOrd >= docOrd) { - // the equals case always means bottom is > doc - // (because we set bottomOrd to the lower bound in - // setBottom): - return 1; - } else { - return -1; - } - } - - @Override - public void copy(int slot, int doc) throws IOException { - final long ord = getRelevantOrd(readerOrds, doc, sortMode); - ords[slot] = ord; - if (ord == 0) { - values[slot] = null; - } else { - assert ord > 0; - if (values[slot] == null) { - values[slot] = new BytesRef(); - } - termsIndex.getValueScratchByOrd(ord, values[slot]); - } - readerGen[slot] = currentReaderGen; - } - - @Override - public int compareDocToValue(int doc, BytesRef value) { - BytesRef docValue = getRelevantValue(termsIndex, doc, sortMode); - if (docValue == null) { - if (value == null) { - return 0; - } - return -1; - } else if (value == null) { - return 1; - } - return docValue.compareTo(value); - } - - } - static BytesRef getRelevantValue(BytesValues.WithOrdinals readerValues, int docId, SortMode sortMode) { BytesValues.Iter iter = readerValues.getIter(docId); if (!iter.hasNext()) { diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java index a7935d28aa8..4ddeae8503d 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java +++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java @@ -19,33 +19,37 @@ package org.elasticsearch.index.fielddata.fieldcomparator; -import java.io.IOException; - import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.index.fielddata.BytesValues; import org.elasticsearch.index.fielddata.IndexFieldData; +import java.io.IOException; + /** * Sorts by field's natural Term sort order. All * comparisons are done using BytesRef.compareTo, which is * slow for medium to large result sets but possibly * very fast for very small results sets. */ -public final class BytesRefValComparator extends FieldComparator { +public final class BytesRefValComparator extends NestedWrappableComparator { private final IndexFieldData indexFieldData; private final SortMode sortMode; + private final BytesRef missingValue; private final BytesRef[] values; private BytesRef bottom; private BytesValues docTerms; - BytesRefValComparator(IndexFieldData indexFieldData, int numHits, SortMode sortMode) { + BytesRefValComparator(IndexFieldData indexFieldData, int numHits, SortMode sortMode, BytesRef missingValue) { this.sortMode = sortMode; values = new BytesRef[numHits]; this.indexFieldData = indexFieldData; + this.missingValue = missingValue; } @Override @@ -57,16 +61,23 @@ public final class BytesRefValComparator extends FieldComparator { @Override public int compareBottom(int doc) throws IOException { - final BytesRef val2 = docTerms.getValue(doc); + BytesRef val2 = docTerms.getValue(doc); + if (val2 == null) { + val2 = missingValue; + } return compareValues(bottom, val2); } @Override public void copy(int slot, int doc) throws IOException { - if (values[slot] == null) { - values[slot] = new BytesRef(); + if (!docTerms.hasValue(doc)) { + values[slot] = missingValue; + } else { + if (values[slot] == null || values[slot] == missingValue) { + values[slot] = new BytesRef(); + } + docTerms.getValueScratch(doc, values[slot]); } - docTerms.getValueScratch(doc, values[slot]); } @Override @@ -143,23 +154,32 @@ public final class BytesRefValComparator extends FieldComparator { } @Override - public BytesRef getValueScratch(int docId, BytesRef scratch) { + public BytesRef getValueScratch(int docId, BytesRef relevantVal) { BytesValues.Iter iter = delegate.getIter(docId); if (!iter.hasNext()) { - return null; + relevantVal.length = 0; + return relevantVal; } BytesRef currentVal = iter.next(); - BytesRef relevantVal = currentVal; + // We MUST allocate a new byte[] since relevantVal might have been filled by reference by a PagedBytes instance + // meaning that the BytesRef.bytes are shared and shouldn't be overwritten. We can't use the bytes of the iterator + // either because they will be overwritten by subsequent calls in the current thread + relevantVal.bytes = new byte[ArrayUtil.oversize(currentVal.length, RamUsageEstimator.NUM_BYTES_BYTE)]; + relevantVal.offset = 0; + relevantVal.length = 0; + relevantVal.append(currentVal); while (true) { int cmp = currentVal.compareTo(relevantVal); if (sortMode == SortMode.MAX) { if (cmp > 0) { - relevantVal = currentVal; + relevantVal.length = 0; + relevantVal.append(currentVal); } } else { if (cmp < 0) { - relevantVal = currentVal; + relevantVal.length = 0; + relevantVal.append(currentVal); } } if (!iter.hasNext()) { @@ -172,4 +192,14 @@ public final class BytesRefValComparator extends FieldComparator { } + @Override + public void missing(int slot) { + values[slot] = missingValue; + } + + @Override + public int compareBottomMissing() { + return compareValues(bottom, missingValue); + } + } diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NestedWrappableComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NestedWrappableComparator.java new file mode 100644 index 00000000000..b521147f640 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NestedWrappableComparator.java @@ -0,0 +1,42 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata.fieldcomparator; + +import org.apache.lucene.search.FieldComparator; + +/** Base comparator which allows for nested sorting. */ +public abstract class NestedWrappableComparator extends FieldComparator { + + /** + * Assigns the underlying missing value to the specified slot, if the actual implementation supports missing value. + * + * @param slot The slot to assign the the missing value to. + */ + public abstract void missing(int slot); + + /** + * Compares the missing value to the bottom. + * + * @return any N < 0 if the bottom value is not competitive with the missing value, any N > 0 if the + * bottom value is competitive with the missing value and 0 if they are equal. + */ + public abstract int compareBottomMissing(); + +} diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NumberComparatorBase.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NumberComparatorBase.java index ae05d9722ee..d7d6939f226 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NumberComparatorBase.java +++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/NumberComparatorBase.java @@ -19,13 +19,12 @@ package org.elasticsearch.index.fielddata.fieldcomparator; -import org.apache.lucene.search.FieldComparator; /** * Base FieldComparator class for number fields. */ // This is right now only used for sorting number based fields inside nested objects -public abstract class NumberComparatorBase extends FieldComparator { +public abstract class NumberComparatorBase extends NestedWrappableComparator { /** * Adds numeric value at the specified doc to the specified slot. @@ -43,18 +42,4 @@ public abstract class NumberComparatorBase extends FieldComparator { */ public abstract void divide(int slot, int divisor); - /** - * Assigns the underlying missing value to the specified slot, if the actual implementation supports missing value. - * - * @param slot The slot to assign the the missing value to. - */ - public abstract void missing(int slot); - - /** - * Compares the missing value to the bottom. - * - * @return any N < 0 if the bottom value is not competitive with the missing value, any N > 0 if the - * bottom value is competitive with the missing value and 0 if they are equal. - */ - public abstract int compareBottomMissing(); } diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBytesIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBytesIndexFieldData.java index 282780b6fe9..ee3c74eb43e 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBytesIndexFieldData.java +++ b/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBytesIndexFieldData.java @@ -59,8 +59,7 @@ public abstract class AbstractBytesIndexFieldData) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits); case AVG: - return new NestedFieldComparator.Avg((NumberComparatorBase) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits); + return new NestedFieldComparator.Avg((NumberComparatorBase) wrappedComparator, rootDocumentsFilter, innerDocumentsFilter, numHits); default: throw new ElasticSearchIllegalArgumentException( String.format(Locale.ROOT, "Unsupported sort_mode[%s] for nested type", sortMode) @@ -194,7 +195,6 @@ abstract class NestedFieldComparator extends FieldComparator { copyMissing(wrappedComparator, slot); return; } - wrappedComparator.copy(spareSlot, nestedDoc); wrappedComparator.copy(slot, nestedDoc); while (true) { @@ -263,7 +263,6 @@ abstract class NestedFieldComparator extends FieldComparator { copyMissing(wrappedComparator, slot); return; } - wrappedComparator.copy(spareSlot, nestedDoc); wrappedComparator.copy(slot, nestedDoc); while (true) { @@ -406,15 +405,15 @@ abstract class NestedFieldComparator extends FieldComparator { } } - static final void copyMissing(FieldComparator comparator, int slot) { - if (comparator instanceof NumberComparatorBase) { - ((NumberComparatorBase) comparator).missing(slot); + static final void copyMissing(FieldComparator comparator, int slot) { + if (comparator instanceof NestedWrappableComparator) { + ((NestedWrappableComparator) comparator).missing(slot); } } - static final int compareBottomMissing(FieldComparator comparator) { - if (comparator instanceof NumberComparatorBase) { - return ((NumberComparatorBase) comparator).compareBottomMissing(); + static final int compareBottomMissing(FieldComparator comparator) { + if (comparator instanceof NestedWrappableComparator) { + return ((NestedWrappableComparator) comparator).compareBottomMissing(); } else { return 0; } diff --git a/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java index e1afa4ff5d3..11799f87ec9 100644 --- a/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java +++ b/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.rest.action.support.RestXContentBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHitField; @@ -58,6 +59,7 @@ import static org.elasticsearch.search.internal.InternalSearchHitField.readSearc public class InternalSearchHit implements SearchHit { private static final Object[] EMPTY_SORT_VALUES = new Object[0]; + private static final Text MAX_TERM_AS_TEXT = new StringAndBytesText(BytesRefFieldComparatorSource.MAX_TERM.utf8ToString()); private transient int docId; @@ -444,7 +446,13 @@ public class InternalSearchHit implements SearchHit { if (sortValues != null && sortValues.length > 0) { builder.startArray(Fields.SORT); for (Object sortValue : sortValues) { - builder.value(sortValue); + if (sortValue != null && sortValue.equals(MAX_TERM_AS_TEXT)) { + // We don't display MAX_TERM in JSON responses in case some clients have UTF-8 parsers that wouldn't accept a + // non-character in the response, even though this is valid UTF-8 + builder.nullValue(); + } else { + builder.value(sortValue); + } } builder.endArray(); } diff --git a/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java b/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java index 8703066a826..7b756b5622c 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java @@ -790,9 +790,9 @@ public class SimpleSortTests extends AbstractSharedClusterTest { assertNoFailures(searchResponse); assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l)); - assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("2")); - assertThat((String) searchResponse.getHits().getAt(1).field("id").value(), equalTo("1")); - assertThat((String) searchResponse.getHits().getAt(2).field("id").value(), equalTo("3")); + assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("1")); + assertThat((String) searchResponse.getHits().getAt(1).field("id").value(), equalTo("3")); + assertThat((String) searchResponse.getHits().getAt(2).field("id").value(), equalTo("2")); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -803,9 +803,9 @@ public class SimpleSortTests extends AbstractSharedClusterTest { assertNoFailures(searchResponse); assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l)); - assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("2")); - assertThat((String) searchResponse.getHits().getAt(1).field("id").value(), equalTo("1")); - assertThat((String) searchResponse.getHits().getAt(2).field("id").value(), equalTo("3")); + assertThat((String) searchResponse.getHits().getAt(0).field("id").value(), equalTo("1")); + assertThat((String) searchResponse.getHits().getAt(1).field("id").value(), equalTo("3")); + assertThat((String) searchResponse.getHits().getAt(2).field("id").value(), equalTo("2")); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -846,7 +846,7 @@ public class SimpleSortTests extends AbstractSharedClusterTest { } @Test - public void testSortMissing() throws Exception { + public void testSortMissingNumbers() throws Exception { prepareCreate("test").addMapping("type1", XContentFactory.jsonBuilder() .startObject() @@ -917,6 +917,90 @@ public class SimpleSortTests extends AbstractSharedClusterTest { assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); } + @Test + public void testSortMissingStrings() throws ElasticSearchException, IOException { + prepareCreate("test").addMapping("type1", + XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("properties") + .startObject("value") + .field("type", "string") + .field("index", "not_analyzed") + .endObject() + .endObject() + .endObject() + .endObject()).execute().actionGet(); + ensureGreen(); + client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() + .field("id", "1") + .field("value", "a") + .endObject()).execute().actionGet(); + + client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject() + .field("id", "2") + .endObject()).execute().actionGet(); + + client().prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject() + .field("id", "1") + .field("value", "c") + .endObject()).execute().actionGet(); + + client().admin().indices().prepareFlush().setRefresh(true).execute().actionGet();try { + Thread.sleep(2000); + } catch (InterruptedException e) { + throw new RuntimeException(); + } + + logger.info("--> sort with no missing (same as missing _last)"); + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC)) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); + + logger.info("--> sort with missing _last"); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("_last")) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); + + logger.info("--> sort with missing _first"); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("_first")) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); + + logger.info("--> sort with missing b"); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("b")) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); + } + @Test public void testIgnoreUnmapped() throws Exception { createIndex("test"); diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataImplTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataImplTests.java new file mode 100644 index 00000000000..c726499dd2e --- /dev/null +++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataImplTests.java @@ -0,0 +1,460 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.unit.index.fielddata; + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.search.*; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lucene.HashedBytesRef; +import org.elasticsearch.index.fielddata.AtomicFieldData; +import org.elasticsearch.index.fielddata.BytesValues; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.index.fielddata.fieldcomparator.SortMode; +import org.junit.Test; + +import static org.hamcrest.Matchers.*; + +public abstract class AbstractFieldDataImplTests extends AbstractFieldDataTests { + + protected String one() { + return "1"; + } + + protected String two() { + return "2"; + } + + protected String three() { + return "3"; + } + + protected String four() { + return "4"; + } + + protected String toString(Object value) { + if (value instanceof BytesRef) { + return ((BytesRef) value).utf8ToString(); + } + return value.toString(); + } + + protected abstract void fillSingleValueAllSet() throws Exception; + + protected abstract void add2SingleValuedDocumentsAndDeleteOneOfThem() throws Exception; + + @Test + public void testDeletedDocs() throws Exception { + add2SingleValuedDocumentsAndDeleteOneOfThem(); + IndexFieldData indexFieldData = getForField("value"); + AtomicReaderContext readerContext = refreshReader(); + AtomicFieldData fieldData = indexFieldData.load(readerContext); + BytesValues values = fieldData.getBytesValues(); + for (int i = 0; i < fieldData.getNumDocs(); ++i) { + assertThat(values.hasValue(i), equalTo(true)); + } + } + + @Test + public void testSingleValueAllSet() throws Exception { + fillSingleValueAllSet(); + IndexFieldData indexFieldData = getForField("value"); + AtomicReaderContext readerContext = refreshReader(); + AtomicFieldData fieldData = indexFieldData.load(readerContext); + assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l)); + + assertThat(fieldData.getNumDocs(), equalTo(3)); + + BytesValues bytesValues = fieldData.getBytesValues(); + + assertThat(bytesValues.isMultiValued(), equalTo(false)); + + assertThat(bytesValues.hasValue(0), equalTo(true)); + assertThat(bytesValues.hasValue(1), equalTo(true)); + assertThat(bytesValues.hasValue(2), equalTo(true)); + + assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two()))); + assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one()))); + assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three()))); + + BytesRef bytesRef = new BytesRef(); + assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two()))); + assertThat(bytesRef, equalTo(new BytesRef(two()))); + assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one()))); + assertThat(bytesRef, equalTo(new BytesRef(one()))); + assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three()))); + assertThat(bytesRef, equalTo(new BytesRef(three()))); + + + BytesValues.Iter bytesValuesIter = bytesValues.getIter(0); + assertThat(bytesValuesIter.hasNext(), equalTo(true)); + assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two()))); + assertThat(bytesValuesIter.hasNext(), equalTo(false)); + + BytesValues hashedBytesValues = fieldData.getBytesValues(); + + assertThat(hashedBytesValues.hasValue(0), equalTo(true)); + assertThat(hashedBytesValues.hasValue(1), equalTo(true)); + assertThat(hashedBytesValues.hasValue(2), equalTo(true)); + + assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two()))); + assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(one()))); + assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three()))); + + BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); + assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two()))); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); + + IndexSearcher searcher = new IndexSearcher(readerContext.reader()); + TopFieldDocs topDocs; + + topDocs = searcher.search(new MatchAllDocsQuery(), 10, + new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN)))); + assertThat(topDocs.totalHits, equalTo(3)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); + assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one())); + assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); + assertThat(toString(((FieldDoc) topDocs.scoreDocs[1]).fields[0]), equalTo(two())); + assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); + assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three())); + + topDocs = searcher.search(new MatchAllDocsQuery(), 10, + new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true))); + assertThat(topDocs.totalHits, equalTo(3)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(2)); + assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); + assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); + } + + private HashedBytesRef convert(BytesValues values, int doc) { + BytesRef ref = new BytesRef(); + return new HashedBytesRef(ref, values.getValueHashed(doc, ref)); + } + + protected abstract void fillSingleValueWithMissing() throws Exception; + + @Test + public void testSingleValueWithMissing() throws Exception { + fillSingleValueWithMissing(); + IndexFieldData indexFieldData = getForField("value"); + AtomicFieldData fieldData = indexFieldData.load(refreshReader()); + assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l)); + + assertThat(fieldData.getNumDocs(), equalTo(3)); + + BytesValues bytesValues = fieldData + .getBytesValues(); + + assertThat(bytesValues.isMultiValued(), equalTo(false)); + + assertThat(bytesValues.hasValue(0), equalTo(true)); + assertThat(bytesValues.hasValue(1), equalTo(false)); + assertThat(bytesValues.hasValue(2), equalTo(true)); + + assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two()))); + assertThat(bytesValues.getValue(1), nullValue()); + assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three()))); + + BytesRef bytesRef = new BytesRef(); + assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two()))); + assertThat(bytesRef, equalTo(new BytesRef(two()))); + assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef())); + assertThat(bytesRef, equalTo(new BytesRef())); + assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three()))); + assertThat(bytesRef, equalTo(new BytesRef(three()))); + + + BytesValues.Iter bytesValuesIter = bytesValues.getIter(0); + assertThat(bytesValuesIter.hasNext(), equalTo(true)); + assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two()))); + assertThat(bytesValuesIter.hasNext(), equalTo(false)); + + bytesValuesIter = bytesValues.getIter(1); + assertThat(bytesValuesIter.hasNext(), equalTo(false)); + + BytesValues hashedBytesValues = fieldData.getBytesValues(); + + assertThat(hashedBytesValues.hasValue(0), equalTo(true)); + assertThat(hashedBytesValues.hasValue(1), equalTo(false)); + assertThat(hashedBytesValues.hasValue(2), equalTo(true)); + + assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two()))); + assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(new BytesRef()))); + assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three()))); + + BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); + assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two()))); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); + + hashedBytesValuesIter = hashedBytesValues.getIter(1); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); + } + + protected abstract void fillMultiValueAllSet() throws Exception; + + @Test + public void testMultiValueAllSet() throws Exception { + fillMultiValueAllSet(); + IndexFieldData indexFieldData = getForField("value"); + AtomicFieldData fieldData = indexFieldData.load(refreshReader()); + assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l)); + + assertThat(fieldData.getNumDocs(), equalTo(3)); + + BytesValues bytesValues = fieldData.getBytesValues(); + + assertThat(bytesValues.isMultiValued(), equalTo(true)); + + assertThat(bytesValues.hasValue(0), equalTo(true)); + assertThat(bytesValues.hasValue(1), equalTo(true)); + assertThat(bytesValues.hasValue(2), equalTo(true)); + + assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two()))); + assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one()))); + assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three()))); + + BytesRef bytesRef = new BytesRef(); + assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two()))); + assertThat(bytesRef, equalTo(new BytesRef(two()))); + assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one()))); + assertThat(bytesRef, equalTo(new BytesRef(one()))); + assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three()))); + assertThat(bytesRef, equalTo(new BytesRef(three()))); + + + BytesValues.Iter bytesValuesIter = bytesValues.getIter(0); + assertThat(bytesValuesIter.hasNext(), equalTo(true)); + assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two()))); + assertThat(bytesValuesIter.hasNext(), equalTo(true)); + assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four()))); + assertThat(bytesValuesIter.hasNext(), equalTo(false)); + + BytesValues hashedBytesValues = fieldData.getBytesValues(); + + assertThat(hashedBytesValues.hasValue(0), equalTo(true)); + assertThat(hashedBytesValues.hasValue(1), equalTo(true)); + assertThat(hashedBytesValues.hasValue(2), equalTo(true)); + + assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two()))); + assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(one()))); + assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three()))); + + BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); + assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two()))); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); + assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(four()))); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); + + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); + TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN)))); + assertThat(topDocs.totalHits, equalTo(3)); + assertThat(topDocs.scoreDocs.length, equalTo(3)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); + assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); + assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); + + topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true))); + assertThat(topDocs.totalHits, equalTo(3)); + assertThat(topDocs.scoreDocs.length, equalTo(3)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); + assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); + assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); + } + + protected abstract void fillMultiValueWithMissing() throws Exception; + + @Test + public void testMultiValueWithMissing() throws Exception { + fillMultiValueWithMissing(); + IndexFieldData indexFieldData = getForField("value"); + AtomicFieldData fieldData = indexFieldData.load(refreshReader()); + assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l)); + + assertThat(fieldData.getNumDocs(), equalTo(3)); + + BytesValues bytesValues = fieldData.getBytesValues(); + + assertThat(bytesValues.isMultiValued(), equalTo(true)); + + assertThat(bytesValues.hasValue(0), equalTo(true)); + assertThat(bytesValues.hasValue(1), equalTo(false)); + assertThat(bytesValues.hasValue(2), equalTo(true)); + + assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two()))); + assertThat(bytesValues.getValue(1), nullValue()); + assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three()))); + + BytesRef bytesRef = new BytesRef(); + assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two()))); + assertThat(bytesRef, equalTo(new BytesRef(two()))); + assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef())); + assertThat(bytesRef, equalTo(new BytesRef())); + assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three()))); + assertThat(bytesRef, equalTo(new BytesRef(three()))); + + + BytesValues.Iter bytesValuesIter = bytesValues.getIter(0); + assertThat(bytesValuesIter.hasNext(), equalTo(true)); + assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two()))); + assertThat(bytesValuesIter.hasNext(), equalTo(true)); + assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four()))); + assertThat(bytesValuesIter.hasNext(), equalTo(false)); + + bytesValuesIter = bytesValues.getIter(1); + assertThat(bytesValuesIter.hasNext(), equalTo(false)); + + BytesValues hashedBytesValues = fieldData.getBytesValues(); + + assertThat(hashedBytesValues.hasValue(0), equalTo(true)); + assertThat(hashedBytesValues.hasValue(1), equalTo(false)); + assertThat(hashedBytesValues.hasValue(2), equalTo(true)); + + assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two()))); + assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(new BytesRef()))); + assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three()))); + + BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); + assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two()))); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); + assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(four()))); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); + + hashedBytesValuesIter = hashedBytesValues.getIter(1); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); + } + + public void testMissingValueForAll() throws Exception { + fillAllMissing(); + IndexFieldData indexFieldData = getForField("value"); + AtomicFieldData fieldData = indexFieldData.load(refreshReader()); + // Some impls (FST) return size 0 and some (PagedBytes) do take size in the case no actual data is loaded + assertThat(fieldData.getMemorySizeInBytes(), greaterThanOrEqualTo(0l)); + + assertThat(fieldData.getNumDocs(), equalTo(3)); + + BytesValues bytesValues = fieldData.getBytesValues(); + + assertThat(bytesValues.isMultiValued(), equalTo(false)); + + assertThat(bytesValues.hasValue(0), equalTo(false)); + assertThat(bytesValues.hasValue(1), equalTo(false)); + assertThat(bytesValues.hasValue(2), equalTo(false)); + + assertThat(bytesValues.getValue(0), nullValue()); + assertThat(bytesValues.getValue(1), nullValue()); + assertThat(bytesValues.getValue(2), nullValue()); + + BytesRef bytesRef = new BytesRef(); + assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef())); + assertThat(bytesRef, equalTo(new BytesRef())); + assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef())); + assertThat(bytesRef, equalTo(new BytesRef())); + assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef())); + assertThat(bytesRef, equalTo(new BytesRef())); + + BytesValues.Iter bytesValuesIter = bytesValues.getIter(0); + assertThat(bytesValuesIter.hasNext(), equalTo(false)); + + bytesValuesIter = bytesValues.getIter(1); + assertThat(bytesValuesIter.hasNext(), equalTo(false)); + + bytesValuesIter = bytesValues.getIter(2); + assertThat(bytesValuesIter.hasNext(), equalTo(false)); + + BytesValues hashedBytesValues = fieldData.getBytesValues(); + + assertThat(hashedBytesValues.hasValue(0), equalTo(false)); + assertThat(hashedBytesValues.hasValue(1), equalTo(false)); + assertThat(hashedBytesValues.hasValue(2), equalTo(false)); + + assertThat(hashedBytesValues.getValue(0), nullValue()); + assertThat(hashedBytesValues.getValue(1), nullValue()); + assertThat(hashedBytesValues.getValue(2), nullValue()); + + BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); + + hashedBytesValuesIter = hashedBytesValues.getIter(1); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); + + hashedBytesValuesIter = hashedBytesValues.getIter(2); + assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); + } + + protected abstract void fillAllMissing() throws Exception; + + @Test + public void testSortMultiValuesFields() throws Exception { + fillExtendedMvSet(); + IndexFieldData indexFieldData = getForField("value"); + + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); + TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, + new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN)))); + assertThat(topDocs.totalHits, equalTo(8)); + assertThat(topDocs.scoreDocs.length, equalTo(8)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(7)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("!08")); + assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("02")); + assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("03")); + assertThat(topDocs.scoreDocs[3].doc, equalTo(3)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("04")); + assertThat(topDocs.scoreDocs[4].doc, equalTo(4)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("06")); + assertThat(topDocs.scoreDocs[5].doc, equalTo(6)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("08")); + assertThat(topDocs.scoreDocs[6].doc, equalTo(1)); + assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(BytesRefFieldComparatorSource.MAX_TERM)); + assertThat(topDocs.scoreDocs[7].doc, equalTo(5)); + assertThat((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(BytesRefFieldComparatorSource.MAX_TERM)); + + topDocs = searcher.search(new MatchAllDocsQuery(), 10, + new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true))); + assertThat(topDocs.totalHits, equalTo(8)); + assertThat(topDocs.scoreDocs.length, equalTo(8)); + assertThat(topDocs.scoreDocs[0].doc, equalTo(6)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("10")); + assertThat(topDocs.scoreDocs[1].doc, equalTo(4)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("08")); + assertThat(topDocs.scoreDocs[2].doc, equalTo(3)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("06")); + assertThat(topDocs.scoreDocs[3].doc, equalTo(0)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("04")); + assertThat(topDocs.scoreDocs[4].doc, equalTo(2)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("03")); + assertThat(topDocs.scoreDocs[5].doc, equalTo(7)); + assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("!10")); + assertThat(topDocs.scoreDocs[6].doc, equalTo(1)); + assertThat(((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(null)); + assertThat(topDocs.scoreDocs[7].doc, equalTo(5)); + assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null)); + } + + protected abstract void fillExtendedMvSet() throws Exception; + +} diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractNumericFieldDataTests.java similarity index 99% rename from src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java rename to src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractNumericFieldDataTests.java index 2328117e9e4..a714c474b8e 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractNumericFieldDataTests.java @@ -33,8 +33,7 @@ import static org.hamcrest.Matchers.equalTo; /** */ -@Ignore("abstract") -public abstract class NumericFieldDataTests extends AbstractStringFieldDataTests { +public abstract class AbstractNumericFieldDataTests extends AbstractFieldDataImplTests { protected abstract FieldDataType getFieldDataType(); diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractStringFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractStringFieldDataTests.java index 085df65c38a..50065db1b1a 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractStringFieldDataTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractStringFieldDataTests.java @@ -19,49 +19,37 @@ package org.elasticsearch.test.unit.index.fielddata; +import com.carrotsearch.randomizedtesting.annotations.Repeat; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.StringField; -import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.*; +import org.apache.lucene.search.join.ScoreMode; +import org.apache.lucene.search.join.ToParentBlockJoinQuery; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.lucene.HashedBytesRef; -import org.elasticsearch.index.fielddata.AtomicFieldData; -import org.elasticsearch.index.fielddata.BytesValues; +import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util._TestUtil; +import org.elasticsearch.common.lucene.search.NotFilter; +import org.elasticsearch.common.lucene.search.TermFilter; +import org.elasticsearch.common.lucene.search.XFilteredQuery; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.fieldcomparator.SortMode; -import org.junit.Test; +import org.elasticsearch.index.search.nested.NestedFieldComparatorSource; -import static org.hamcrest.Matchers.*; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; /** */ -public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTests { - - protected String one() { - return "1"; - } - - protected String two() { - return "2"; - } - - protected String three() { - return "3"; - } - - protected String four() { - return "4"; - } - - private String toString(Object value) { - if (value instanceof BytesRef) { - return ((BytesRef) value).utf8ToString(); - } - return value.toString(); - } +public abstract class AbstractStringFieldDataTests extends AbstractFieldDataImplTests { protected void fillSingleValueAllSet() throws Exception { Document d = new Document(); @@ -96,100 +84,6 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest writer.deleteDocuments(new Term("_id", "1")); } - @Test - public void testDeletedDocs() throws Exception { - add2SingleValuedDocumentsAndDeleteOneOfThem(); - IndexFieldData indexFieldData = getForField("value"); - AtomicReaderContext readerContext = refreshReader(); - AtomicFieldData fieldData = indexFieldData.load(readerContext); - assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l)); - BytesValues values = fieldData.getBytesValues(); - assertThat(fieldData.getNumDocs(), equalTo(2)); - assertThat(fieldData.getNumberUniqueValues(), equalTo(2l)); - for (int i = 0; i < fieldData.getNumDocs(); ++i) { - assertThat(values.hasValue(i), equalTo(true)); - } - } - - @Test - public void testSingleValueAllSet() throws Exception { - fillSingleValueAllSet(); - IndexFieldData indexFieldData = getForField("value"); - AtomicReaderContext readerContext = refreshReader(); - AtomicFieldData fieldData = indexFieldData.load(readerContext); - assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l)); - assertThat(fieldData.getNumberUniqueValues(), equalTo(3l)); - assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l)); - - assertThat(fieldData.getNumDocs(), equalTo(3)); - - BytesValues bytesValues = fieldData.getBytesValues(); - - assertThat(bytesValues.isMultiValued(), equalTo(false)); - - assertThat(bytesValues.hasValue(0), equalTo(true)); - assertThat(bytesValues.hasValue(1), equalTo(true)); - assertThat(bytesValues.hasValue(2), equalTo(true)); - - assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two()))); - assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one()))); - assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three()))); - - BytesRef bytesRef = new BytesRef(); - assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two()))); - assertThat(bytesRef, equalTo(new BytesRef(two()))); - assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one()))); - assertThat(bytesRef, equalTo(new BytesRef(one()))); - assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three()))); - assertThat(bytesRef, equalTo(new BytesRef(three()))); - - - BytesValues.Iter bytesValuesIter = bytesValues.getIter(0); - assertThat(bytesValuesIter.hasNext(), equalTo(true)); - assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two()))); - assertThat(bytesValuesIter.hasNext(), equalTo(false)); - - BytesValues hashedBytesValues = fieldData.getBytesValues(); - - assertThat(hashedBytesValues.hasValue(0), equalTo(true)); - assertThat(hashedBytesValues.hasValue(1), equalTo(true)); - assertThat(hashedBytesValues.hasValue(2), equalTo(true)); - - assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two()))); - assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(one()))); - assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three()))); - - BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); - assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two()))); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); - - IndexSearcher searcher = new IndexSearcher(readerContext.reader()); - TopFieldDocs topDocs; - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN)))); - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); - assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one())); - assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); - assertThat(toString(((FieldDoc) topDocs.scoreDocs[1]).fields[0]), equalTo(two())); - assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); - assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three())); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true))); - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(2)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); - } - - private HashedBytesRef convert(BytesValues values, int doc) { - BytesRef ref = new BytesRef(); - return new HashedBytesRef(ref, values.getValueHashed(doc, ref)); - } - protected void fillSingleValueWithMissing() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); @@ -207,68 +101,6 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest writer.addDocument(d); } - @Test - public void testSingleValueWithMissing() throws Exception { - fillSingleValueWithMissing(); - IndexFieldData indexFieldData = getForField("value"); - AtomicFieldData fieldData = indexFieldData.load(refreshReader()); - assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l)); - - assertThat(fieldData.getNumberUniqueValues(), equalTo(2l)); - assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l)); - assertThat(fieldData.getNumDocs(), equalTo(3)); - - BytesValues bytesValues = fieldData - .getBytesValues(); - - assertThat(bytesValues.isMultiValued(), equalTo(false)); - - assertThat(bytesValues.hasValue(0), equalTo(true)); - assertThat(bytesValues.hasValue(1), equalTo(false)); - assertThat(bytesValues.hasValue(2), equalTo(true)); - - assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two()))); - assertThat(bytesValues.getValue(1), nullValue()); - assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three()))); - - BytesRef bytesRef = new BytesRef(); - assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two()))); - assertThat(bytesRef, equalTo(new BytesRef(two()))); - assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef())); - assertThat(bytesRef, equalTo(new BytesRef())); - assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three()))); - assertThat(bytesRef, equalTo(new BytesRef(three()))); - - - BytesValues.Iter bytesValuesIter = bytesValues.getIter(0); - assertThat(bytesValuesIter.hasNext(), equalTo(true)); - assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two()))); - assertThat(bytesValuesIter.hasNext(), equalTo(false)); - - bytesValuesIter = bytesValues.getIter(1); - assertThat(bytesValuesIter.hasNext(), equalTo(false)); - - BytesValues hashedBytesValues = fieldData.getBytesValues(); - - assertThat(hashedBytesValues.hasValue(0), equalTo(true)); - assertThat(hashedBytesValues.hasValue(1), equalTo(false)); - assertThat(hashedBytesValues.hasValue(2), equalTo(true)); - - assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two()))); - assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(new BytesRef()))); - assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three()))); - - BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); - assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two()))); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); - - hashedBytesValuesIter = hashedBytesValues.getIter(1); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); - - // TODO properly support missing.... - } - protected void fillMultiValueAllSet() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); @@ -288,78 +120,6 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest writer.addDocument(d); } - @Test - public void testMultiValueAllSet() throws Exception { - fillMultiValueAllSet(); - IndexFieldData indexFieldData = getForField("value"); - AtomicFieldData fieldData = indexFieldData.load(refreshReader()); - assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l)); - - assertThat(fieldData.getNumDocs(), equalTo(3)); - assertThat(fieldData.getNumberUniqueValues(), equalTo(4l)); - assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l)); - - BytesValues bytesValues = fieldData.getBytesValues(); - - assertThat(bytesValues.isMultiValued(), equalTo(true)); - - assertThat(bytesValues.hasValue(0), equalTo(true)); - assertThat(bytesValues.hasValue(1), equalTo(true)); - assertThat(bytesValues.hasValue(2), equalTo(true)); - - assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two()))); - assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one()))); - assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three()))); - - BytesRef bytesRef = new BytesRef(); - assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two()))); - assertThat(bytesRef, equalTo(new BytesRef(two()))); - assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one()))); - assertThat(bytesRef, equalTo(new BytesRef(one()))); - assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three()))); - assertThat(bytesRef, equalTo(new BytesRef(three()))); - - - BytesValues.Iter bytesValuesIter = bytesValues.getIter(0); - assertThat(bytesValuesIter.hasNext(), equalTo(true)); - assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two()))); - assertThat(bytesValuesIter.hasNext(), equalTo(true)); - assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four()))); - assertThat(bytesValuesIter.hasNext(), equalTo(false)); - - BytesValues hashedBytesValues = fieldData.getBytesValues(); - - assertThat(hashedBytesValues.hasValue(0), equalTo(true)); - assertThat(hashedBytesValues.hasValue(1), equalTo(true)); - assertThat(hashedBytesValues.hasValue(2), equalTo(true)); - - assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two()))); - assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(one()))); - assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three()))); - - BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); - assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two()))); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); - assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(four()))); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); - - IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); - TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN)))); - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs.length, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true))); - assertThat(topDocs.totalHits, equalTo(3)); - assertThat(topDocs.scoreDocs.length, equalTo(3)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(2)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(1)); - } - protected void fillMultiValueWithMissing() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); @@ -378,129 +138,6 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest writer.addDocument(d); } - @Test - public void testMultiValueWithMissing() throws Exception { - fillMultiValueWithMissing(); - IndexFieldData indexFieldData = getForField("value"); - AtomicFieldData fieldData = indexFieldData.load(refreshReader()); - assertThat(fieldData.getMemorySizeInBytes(), greaterThan(0l)); - - assertThat(fieldData.getNumDocs(), equalTo(3)); - assertThat(fieldData.getNumberUniqueValues(), equalTo(3l)); - assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), greaterThan(0l)); - - BytesValues bytesValues = fieldData.getBytesValues(); - - assertThat(bytesValues.isMultiValued(), equalTo(true)); - - assertThat(bytesValues.hasValue(0), equalTo(true)); - assertThat(bytesValues.hasValue(1), equalTo(false)); - assertThat(bytesValues.hasValue(2), equalTo(true)); - - assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two()))); - assertThat(bytesValues.getValue(1), nullValue()); - assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three()))); - - BytesRef bytesRef = new BytesRef(); - assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two()))); - assertThat(bytesRef, equalTo(new BytesRef(two()))); - assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef())); - assertThat(bytesRef, equalTo(new BytesRef())); - assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three()))); - assertThat(bytesRef, equalTo(new BytesRef(three()))); - - - BytesValues.Iter bytesValuesIter = bytesValues.getIter(0); - assertThat(bytesValuesIter.hasNext(), equalTo(true)); - assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two()))); - assertThat(bytesValuesIter.hasNext(), equalTo(true)); - assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four()))); - assertThat(bytesValuesIter.hasNext(), equalTo(false)); - - bytesValuesIter = bytesValues.getIter(1); - assertThat(bytesValuesIter.hasNext(), equalTo(false)); - - BytesValues hashedBytesValues = fieldData.getBytesValues(); - - assertThat(hashedBytesValues.hasValue(0), equalTo(true)); - assertThat(hashedBytesValues.hasValue(1), equalTo(false)); - assertThat(hashedBytesValues.hasValue(2), equalTo(true)); - - assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two()))); - assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(new BytesRef()))); - assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three()))); - - BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); - assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two()))); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(true)); - assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(four()))); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); - - hashedBytesValuesIter = hashedBytesValues.getIter(1); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); - } - - public void testMissingValueForAll() throws Exception { - fillAllMissing(); - IndexFieldData indexFieldData = getForField("value"); - AtomicFieldData fieldData = indexFieldData.load(refreshReader()); - // Some impls (FST) return size 0 and some (PagedBytes) do take size in the case no actual data is loaded - assertThat(fieldData.getMemorySizeInBytes(), greaterThanOrEqualTo(0l)); - - assertThat(fieldData.getNumDocs(), equalTo(3)); - assertThat(fieldData.getNumberUniqueValues(), equalTo(0l)); - assertThat(indexFieldData.getHighestNumberOfSeenUniqueValues(), equalTo(0l)); - - BytesValues bytesValues = fieldData.getBytesValues(); - - assertThat(bytesValues.isMultiValued(), equalTo(false)); - - assertThat(bytesValues.hasValue(0), equalTo(false)); - assertThat(bytesValues.hasValue(1), equalTo(false)); - assertThat(bytesValues.hasValue(2), equalTo(false)); - - assertThat(bytesValues.getValue(0), nullValue()); - assertThat(bytesValues.getValue(1), nullValue()); - assertThat(bytesValues.getValue(2), nullValue()); - - BytesRef bytesRef = new BytesRef(); - assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef())); - assertThat(bytesRef, equalTo(new BytesRef())); - assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef())); - assertThat(bytesRef, equalTo(new BytesRef())); - assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef())); - assertThat(bytesRef, equalTo(new BytesRef())); - - BytesValues.Iter bytesValuesIter = bytesValues.getIter(0); - assertThat(bytesValuesIter.hasNext(), equalTo(false)); - - bytesValuesIter = bytesValues.getIter(1); - assertThat(bytesValuesIter.hasNext(), equalTo(false)); - - bytesValuesIter = bytesValues.getIter(2); - assertThat(bytesValuesIter.hasNext(), equalTo(false)); - - BytesValues hashedBytesValues = fieldData.getBytesValues(); - - assertThat(hashedBytesValues.hasValue(0), equalTo(false)); - assertThat(hashedBytesValues.hasValue(1), equalTo(false)); - assertThat(hashedBytesValues.hasValue(2), equalTo(false)); - - assertThat(hashedBytesValues.getValue(0), nullValue()); - assertThat(hashedBytesValues.getValue(1), nullValue()); - assertThat(hashedBytesValues.getValue(2), nullValue()); - - BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); - - hashedBytesValuesIter = hashedBytesValues.getIter(1); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); - - hashedBytesValuesIter = hashedBytesValues.getIter(2); - assertThat(hashedBytesValuesIter.hasNext(), equalTo(false)); - } - protected void fillAllMissing() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); @@ -515,55 +152,6 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest writer.addDocument(d); } - @Test - public void testSortMultiValuesFields() throws Exception { - fillExtendedMvSet(); - IndexFieldData indexFieldData = getForField("value"); - - IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); - TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN)))); - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(1)); - assertThat(((FieldDoc) topDocs.scoreDocs[0]).fields[0], equalTo(null)); - assertThat(topDocs.scoreDocs[1].doc, equalTo(5)); - assertThat(((FieldDoc) topDocs.scoreDocs[1]).fields[0], equalTo(null)); - assertThat(topDocs.scoreDocs[2].doc, equalTo(7)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("!08")); - assertThat(topDocs.scoreDocs[3].doc, equalTo(0)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("02")); - assertThat(topDocs.scoreDocs[4].doc, equalTo(2)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("03")); - assertThat(topDocs.scoreDocs[5].doc, equalTo(3)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("04")); - assertThat(topDocs.scoreDocs[6].doc, equalTo(4)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("06")); - assertThat(topDocs.scoreDocs[7].doc, equalTo(6)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]).utf8ToString(), equalTo("08")); - - topDocs = searcher.search(new MatchAllDocsQuery(), 10, - new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true))); - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(6)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("10")); - assertThat(topDocs.scoreDocs[1].doc, equalTo(4)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("08")); - assertThat(topDocs.scoreDocs[2].doc, equalTo(3)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("06")); - assertThat(topDocs.scoreDocs[3].doc, equalTo(0)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("04")); - assertThat(topDocs.scoreDocs[4].doc, equalTo(2)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("03")); - assertThat(topDocs.scoreDocs[5].doc, equalTo(7)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("!10")); - assertThat(topDocs.scoreDocs[6].doc, equalTo(1)); - assertThat(((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(null)); - assertThat(topDocs.scoreDocs[7].doc, equalTo(5)); - assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null)); - } - protected void fillExtendedMvSet() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); @@ -615,4 +203,224 @@ public abstract class AbstractStringFieldDataTests extends AbstractFieldDataTest writer.addDocument(d); } + @Repeat(iterations=10) + public void testActualMissingValue() throws IOException { + testActualMissingValue(false); + } + + @Repeat(iterations=10) + public void testActualMissingValueReverse() throws IOException { + testActualMissingValue(true); + } + + public void testActualMissingValue(boolean reverse) throws IOException { + // missing value is set to an actual value + Document d = new Document(); + final StringField s = new StringField("value", "", Field.Store.YES); + d.add(s); + final String[] values = new String[randomIntBetween(2, 30)]; + for (int i = 1; i < values.length; ++i) { + values[i] = _TestUtil.randomUnicodeString(getRandom()); + } + final int numDocs = atLeast(100); + for (int i = 0; i < numDocs; ++i) { + final String value = RandomPicks.randomFrom(getRandom(), values); + if (value == null) { + writer.addDocument(new Document()); + } else { + s.setStringValue(value); + writer.addDocument(d); + } + if (randomInt(10) == 0) { + writer.commit(); + } + } + + final IndexFieldData indexFieldData = getForField("value"); + final String missingValue = values[1]; + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); + XFieldComparatorSource comparator = indexFieldData.comparatorSource(missingValue, SortMode.MIN); + TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse))); + assertEquals(numDocs, topDocs.totalHits); + BytesRef previousValue = reverse ? UnicodeUtil.BIG_TERM : new BytesRef(); + for (int i = 0; i < topDocs.scoreDocs.length; ++i) { + final String docValue = searcher.doc(topDocs.scoreDocs[i].doc).get("value"); + final BytesRef value = new BytesRef(docValue == null ? missingValue : docValue); + if (reverse) { + assertTrue(previousValue.compareTo(value) >= 0); + } else { + assertTrue(previousValue.compareTo(value) <= 0); + } + previousValue = value; + } + searcher.getIndexReader().close(); + } + + @Repeat(iterations=3) + public void testSortMissingFirst() throws IOException { + testSortMissing(true, false); + } + + @Repeat(iterations=3) + public void testSortMissingFirstReverse() throws IOException { + testSortMissing(true, true); + } + + @Repeat(iterations=3) + public void testSortMissingLast() throws IOException { + testSortMissing(false, false); + } + + @Repeat(iterations=3) + public void testSortMissingLastReverse() throws IOException { + testSortMissing(false, true); + } + + public void testSortMissing(boolean first, boolean reverse) throws IOException { + Document d = new Document(); + final StringField s = new StringField("value", "", Field.Store.YES); + d.add(s); + final String[] values = new String[randomIntBetween(2, 10)]; + for (int i = 1; i < values.length; ++i) { + values[i] = _TestUtil.randomUnicodeString(getRandom()); + } + final int numDocs = atLeast(100); + for (int i = 0; i < numDocs; ++i) { + final String value = RandomPicks.randomFrom(getRandom(), values); + if (value == null) { + writer.addDocument(new Document()); + } else { + s.setStringValue(value); + writer.addDocument(d); + } + if (randomInt(10) == 0) { + writer.commit(); + } + } + final IndexFieldData indexFieldData = getForField("value"); + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); + XFieldComparatorSource comparator = indexFieldData.comparatorSource(first ? "_first" : "_last", SortMode.MIN); + TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(new SortField("value", comparator, reverse))); + assertEquals(numDocs, topDocs.totalHits); + BytesRef previousValue = first ? null : reverse ? UnicodeUtil.BIG_TERM : new BytesRef(); + for (int i = 0; i < topDocs.scoreDocs.length; ++i) { + final String docValue = searcher.doc(topDocs.scoreDocs[i].doc).get("value"); + if (first && docValue == null) { + assertNull(previousValue); + } else if (!first && docValue != null) { + assertNotNull(previousValue); + } + final BytesRef value = docValue == null ? null : new BytesRef(docValue); + if (previousValue != null && value != null) { + if (reverse) { + assertTrue(previousValue.compareTo(value) >= 0); + } else { + assertTrue(previousValue.compareTo(value) <= 0); + } + } + previousValue = value; + } + searcher.getIndexReader().close(); + } + + @Repeat(iterations=3) + public void testNestedSortingMin() throws IOException { + testNestedSorting(SortMode.MIN); + } + + @Repeat(iterations=3) + public void testNestedSortingMax() throws IOException { + testNestedSorting(SortMode.MAX); + } + + public void testNestedSorting(SortMode sortMode) throws IOException { + final String[] values = new String[randomIntBetween(2, 20)]; + for (int i = 0; i < values.length; ++i) { + values[i] = _TestUtil.randomSimpleString(getRandom()); + } + final int numParents = atLeast(100); + List docs = new ArrayList(); + final OpenBitSet parents = new OpenBitSet(); + for (int i = 0; i < numParents; ++i) { + docs.clear(); + final int numChildren = randomInt(4); + for (int j = 0; j < numChildren; ++j) { + final Document child = new Document(); + final int numValues = randomInt(3); + for (int k = 0; k < numValues; ++k) { + final String value = RandomPicks.randomFrom(getRandom(), values); + child.add(new StringField("text", value, Store.YES)); + } + docs.add(child); + } + final Document parent = new Document(); + parent.add(new StringField("type", "parent", Store.YES)); + final String value = RandomPicks.randomFrom(getRandom(), values); + if (value != null) { + parent.add(new StringField("text", value, Store.YES)); + } + docs.add(parent); + parents.set(parents.prevSetBit(parents.length() - 1) + docs.size()); + writer.addDocuments(docs); + if (randomInt(10) == 0) { + writer.commit(); + } + } + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); + IndexFieldData fieldData = getForField("text"); + final BytesRef missingValue; + switch (randomInt(4)) { + case 0: + missingValue = new BytesRef(); + break; + case 1: + missingValue = BytesRefFieldComparatorSource.MAX_TERM; + break; + case 2: + missingValue = new BytesRef(RandomPicks.randomFrom(getRandom(), values)); + break; + default: + missingValue = new BytesRef(_TestUtil.randomSimpleString(getRandom())); + break; + } + BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode); + Filter parentFilter = new TermFilter(new Term("type", "parent")); + Filter childFilter = new NotFilter(parentFilter); + NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter); + ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new XFilteredQuery(new MatchAllDocsQuery(), childFilter), new CachingWrapperFilter(parentFilter), ScoreMode.None); + Sort sort = new Sort(new SortField("text", nestedComparatorSource)); + TopFieldDocs topDocs = searcher.search(query, randomIntBetween(1, numParents), sort); + assertTrue(topDocs.scoreDocs.length > 0); + BytesRef previous = null; + for (int i = 0; i < topDocs.scoreDocs.length; ++i) { + final int docID = topDocs.scoreDocs[i].doc; + assert parents.get(docID); + BytesRef cmpValue = null; + for (int child = parents.prevSetBit(docID - 1) + 1; child < docID; ++child) { + String[] vals = searcher.doc(child).getValues("text"); + if (vals.length == 0) { + vals = new String[] {missingValue.utf8ToString()}; + } + for (String value : vals) { + final BytesRef bytesValue = new BytesRef(value); + if (cmpValue == null) { + cmpValue = bytesValue; + } else if (sortMode == SortMode.MIN && bytesValue.compareTo(cmpValue) < 0) { + cmpValue = bytesValue; + } else if (sortMode == SortMode.MAX && bytesValue.compareTo(cmpValue) > 0) { + cmpValue = bytesValue; + } + } + } + if (cmpValue == null) { + cmpValue = missingValue; + } + if (previous != null) { + assertNotNull(cmpValue); + assertTrue(previous.utf8ToString() + " / " + cmpValue.utf8ToString(), previous.compareTo(cmpValue) <= 0); + } + previous = cmpValue; + } + searcher.getIndexReader().close(); + } } diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java index 86fa7bd31cc..cc22ef9bfc1 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java @@ -28,7 +28,7 @@ import org.elasticsearch.index.fielddata.FieldDataType; /** */ -public class DoubleFieldDataTests extends NumericFieldDataTests { +public class DoubleFieldDataTests extends AbstractNumericFieldDataTests { @Override protected FieldDataType getFieldDataType() { diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/FloatFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/FloatFieldDataTests.java index 65f0f226125..7813b63462a 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/FloatFieldDataTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/FloatFieldDataTests.java @@ -28,7 +28,7 @@ import org.elasticsearch.index.fielddata.FieldDataType; /** */ -public class FloatFieldDataTests extends NumericFieldDataTests { +public class FloatFieldDataTests extends AbstractNumericFieldDataTests { @Override protected FieldDataType getFieldDataType() { diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/LongFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/LongFieldDataTests.java index 3faa89ec078..7b5708c76ff 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/fielddata/LongFieldDataTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/LongFieldDataTests.java @@ -46,7 +46,7 @@ import static org.hamcrest.Matchers.instanceOf; /** * Tests for all integer types (byte, short, int, long). */ -public class LongFieldDataTests extends NumericFieldDataTests { +public class LongFieldDataTests extends AbstractNumericFieldDataTests { @Override protected FieldDataType getFieldDataType() { diff --git a/src/test/java/org/elasticsearch/test/unit/index/fielddata/NoOrdinalsStringFieldDataTests.java b/src/test/java/org/elasticsearch/test/unit/index/fielddata/NoOrdinalsStringFieldDataTests.java new file mode 100644 index 00000000000..e2669437ddc --- /dev/null +++ b/src/test/java/org/elasticsearch/test/unit/index/fielddata/NoOrdinalsStringFieldDataTests.java @@ -0,0 +1,90 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.unit.index.fielddata; + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.fielddata.AtomicFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.index.fielddata.fieldcomparator.SortMode; +import org.elasticsearch.index.mapper.FieldMapper.Names; + +/** Returns an implementation based on paged bytes which doesn't implement WithOrdinals in order to visit different paths in the code, + * eg. BytesRefFieldComparatorSource makes decisions based on whether the field data implements WithOrdinals. */ +public class NoOrdinalsStringFieldDataTests extends PagedBytesStringFieldDataTests { + + @SuppressWarnings("unchecked") + @Override + public IndexFieldData> getForField(String fieldName) { + final IndexFieldData in = super.getForField(fieldName); + return new IndexFieldData>() { + + @Override + public Index index() { + return in.index(); + } + + @Override + public Names getFieldNames() { + return in.getFieldNames(); + } + + @Override + public boolean valuesOrdered() { + return in.valuesOrdered(); + } + + @Override + public AtomicFieldData load(AtomicReaderContext context) { + return in.load(context); + } + + @Override + public AtomicFieldData loadDirect(AtomicReaderContext context) throws Exception { + return in.loadDirect(context); + } + + @Override + public XFieldComparatorSource comparatorSource(Object missingValue, SortMode sortMode) { + return new BytesRefFieldComparatorSource(this, missingValue, sortMode); + } + + @Override + public void clear() { + in.clear(); + } + + @Override + public void clear(IndexReader reader) { + in.clear(reader); + } + + @Override + public long getHighestNumberOfSeenUniqueValues() { + return in.getHighestNumberOfSeenUniqueValues(); + } + + }; + } + +} diff --git a/src/test/java/org/elasticsearch/test/unit/index/search/nested/NestedSortingTests.java b/src/test/java/org/elasticsearch/test/unit/index/search/nested/NestedSortingTests.java index 2e4f51060ed..f3e05648666 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/search/nested/NestedSortingTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/search/nested/NestedSortingTests.java @@ -46,7 +46,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; /** @@ -216,7 +215,7 @@ public class NestedSortingTests extends AbstractFieldDataTests { SortMode sortMode = SortMode.MIN; IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false)); PagedBytesIndexFieldData indexFieldData = getForField("field2"); - BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(indexFieldData, sortMode); + BytesRefFieldComparatorSource innerSource = new BytesRefFieldComparatorSource(indexFieldData, null, sortMode); Filter parentFilter = new TermFilter(new Term("__type", "parent")); Filter childFilter = new NotFilter(parentFilter); NestedFieldComparatorSource nestedComparatorSource = new NestedFieldComparatorSource(sortMode, innerSource, parentFilter, childFilter);