Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8592: Fix index sorting corruption due to numeric overflow 

The merge sort of sorted segments can produce an invalid
sort if the sort field is an Integer/Long that uses reverse order and contains values equal to
Integer/Long#MIN_VALUE. These values are always sorted first during a merge
(instead of last because of the reverse order) due to this bug.
Indices affected by the bug can be detected by running the CheckIndex command on a
distribution that contains the fix (7.6+).
This commit is contained in:
Jim Ferenczi 2018-12-07 19:16:42 +01:00
parent da7919f7a4
commit 9f29ed0757
5 changed files with 941 additions and 705 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lucene/CHANGES.txt
View File

@ -314,7 +314,14 @@ Bug fixes
* LUCENE-8595: Fix interleaved DV update and reset. Interleaved update and reset value
to the same doc in the same updates package looses an update if the reset comes before
the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grant)
the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grand)
* LUCENE-8592: Fix index sorting corruption due to numeric overflow. The merge of sorted segments
can produce an invalid sort if the sort field is an Integer/Long that uses reverse order and contains
values equal to Integer/Long#MIN_VALUE. These values are always sorted first during a merge
(instead of last because of the reverse order) due to this bug. Indices affected by the bug can be
detected by running the CheckIndex command on a distribution that contains the fix (7.6+).
(Jim Ferenczi, Adrien Grand, Mike McCandless, Simon Willnauer)
New Features

27
lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
View File

@ -1718,6 +1718,33 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
}
/**
* Tests that {@link CheckIndex} can detect invalid sort on sorted indices created
* before https://issues.apache.org/jira/browse/LUCENE-8592.
*/
public void testSortedIndexWithInvalidSort() throws Exception {
Path path = createTempDir("sorted");
String name = "sorted-invalid.7.5.0.zip";
InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name);
assertNotNull("Sorted index index " + name + " not found", resource);
TestUtil.unzip(resource, path);
Directory dir = FSDirectory.open(path);
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.leaves().size());
Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
assertNotNull(sort);
assertEquals("<long: \"dateDV\">! missingValue=-9223372036854775808", sort.toString());
reader.close();
CheckIndex.Status status = TestUtil.checkIndex(dir);
assertEquals(1, status.segmentInfos.size());
assertNotNull(status.segmentInfos.get(0).indexSortStatus.error);
assertEquals(status.segmentInfos.get(0).indexSortStatus.error.getMessage(),
"segment has indexSort=<long: \"dateDV\">! missingValue=-9223372036854775808 but docID=4 sorts after docID=5");
dir.close();
}
static long getValue(BinaryDocValues bdv) throws IOException {
BytesRef term = bdv.binaryValue();
int idx = term.offset;

BIN
lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip Normal file
View File

Binary file not shown.

30
lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
View File

@ -42,17 +42,18 @@ final class MultiSorter {
SortField fields[] = sort.getSort();
final ComparableProvider[][] comparables = new ComparableProvider[fields.length][];
final int[] reverseMuls = new int[fields.length];
for(int i=0;i<fields.length;i++) {
comparables[i] = getComparableProviders(readers, fields[i]);
reverseMuls[i] = fields[i].getReverse() ? -1 : 1;
}
int leafCount = readers.size();
PriorityQueue<LeafAndDocID> queue = new PriorityQueue<LeafAndDocID>(leafCount) {
@Override
public boolean lessThan(LeafAndDocID a, LeafAndDocID b) {
for(int i=0;i<comparables.length;i++) {
int cmp = a.values[i].compareTo(b.values[i]);
int cmp = reverseMuls[i] * a.values[i].compareTo(b.values[i]);
if (cmp != 0) {
return cmp < 0;
}
@ -146,14 +147,13 @@ final class MultiSorter {
/** Returns an object for this docID whose .compareTo represents the requested {@link SortField} sort order. */
private interface ComparableProvider {
public Comparable getComparable(int docID) throws IOException;
Comparable getComparable(int docID) throws IOException;
}
/** Returns {@code ComparableProvider}s for the provided readers to represent the requested {@link SortField} sort order. */
private static ComparableProvider[] getComparableProviders(List<CodecReader> readers, SortField sortField) throws IOException {
ComparableProvider[] providers = new ComparableProvider[readers.size()];
final int reverseMul = sortField.getReverse() ? -1 : 1;
final SortField.Type sortType = Sorter.getSortFieldType(sortField);
switch(sortType) {
@ -169,9 +169,9 @@ final class MultiSorter {
OrdinalMap ordinalMap = OrdinalMap.build(null, values, PackedInts.DEFAULT);
final int missingOrd;
if (sortField.getMissingValue() == SortField.STRING_LAST) {
missingOrd = sortField.getReverse() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
missingOrd = Integer.MAX_VALUE;
} else {
missingOrd = sortField.getReverse() ? Integer.MAX_VALUE : Integer.MIN_VALUE;
missingOrd = Integer.MIN_VALUE;
}
for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
@ -197,7 +197,7 @@ final class MultiSorter {
}
if (readerDocID == docID) {
// translate segment's ord to global ord space:
return reverseMul * (int) globalOrds.get(readerValues.ordValue());
return Math.toIntExact(globalOrds.get(readerValues.ordValue()));
} else {
return missingOrd;
}
@ -238,9 +238,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
return reverseMul * values.longValue();
return values.longValue();
} else {
return reverseMul * missingValue;
return missingValue;
}
}
};
@ -279,9 +279,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
return reverseMul * (int) values.longValue();
return (int) values.longValue();
} else {
return reverseMul * missingValue;
return missingValue;
}
}
};
@ -320,9 +320,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
return reverseMul * Double.longBitsToDouble(values.longValue());
return Double.longBitsToDouble(values.longValue());
} else {
return reverseMul * missingValue;
return missingValue;
}
}
};
@ -361,9 +361,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
return reverseMul * Float.intBitsToFloat((int) values.longValue());
return Float.intBitsToFloat((int) values.longValue());
} else {
return reverseMul * missingValue;
return missingValue;
}
}
};

242
lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
View File

@ -355,9 +355,10 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingStringFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.STRING);
SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
sortField.setMissingValue(SortField.STRING_FIRST);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -381,20 +382,29 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
SortedDocValues values = leaf.getSortedDocValues("foo");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals("zzz", values.binaryValue().utf8ToString());
assertEquals(1, values.nextDoc());
assertEquals("mmm", values.binaryValue().utf8ToString());
} else {
// docID 0 is missing:
assertEquals(1, values.nextDoc());
assertEquals("mmm", values.binaryValue().utf8ToString());
assertEquals(2, values.nextDoc());
assertEquals("zzz", values.binaryValue().utf8ToString());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedStringFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedSetSortField("foo", false);
SortField sortField = new SortedSetSortField("foo", reverse);
sortField.setMissingValue(SortField.STRING_FIRST);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -425,21 +435,32 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3l, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2l, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1l, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1l, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2l, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3l, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingStringLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.STRING);
SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
sortField.setMissingValue(SortField.STRING_LAST);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -463,20 +484,29 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
SortedDocValues values = leaf.getSortedDocValues("foo");
if (reverse) {
assertEquals(1, values.nextDoc());
assertEquals("zzz", values.binaryValue().utf8ToString());
assertEquals(2, values.nextDoc());
assertEquals("mmm", values.binaryValue().utf8ToString());
} else {
assertEquals(0, values.nextDoc());
assertEquals("mmm", values.binaryValue().utf8ToString());
assertEquals(1, values.nextDoc());
assertEquals("zzz", values.binaryValue().utf8ToString());
}
assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedStringLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedSetSortField("foo", false);
SortField sortField = new SortedSetSortField("foo", reverse);
sortField.setMissingValue(SortField.STRING_LAST);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -506,16 +536,26 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3l, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2l, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1l, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1l, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2l, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3l, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testBasicLong() throws Exception {
Directory dir = newDirectory();
@ -597,9 +637,10 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingLongFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.LONG);
SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -623,20 +664,29 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(18, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(7, values.longValue());
} else {
// docID 0 has no value
assertEquals(1, values.nextDoc());
assertEquals(7, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(18, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedLongFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -666,21 +716,32 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingLongLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.LONG);
SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -704,20 +765,30 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
// docID 0 is missing
assertEquals(1, values.nextDoc());
assertEquals(18, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(7, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(7, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(18, values.longValue());
}
assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedLongLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -748,16 +819,26 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testBasicInt() throws Exception {
Directory dir = newDirectory();
@ -841,9 +922,10 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingIntFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.INT);
SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -867,19 +949,28 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(18, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(7, values.longValue());
} else {
assertEquals(1, values.nextDoc());
assertEquals(7, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(18, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedIntFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -909,21 +1000,32 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingIntLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.INT);
SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -947,20 +1049,30 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
// docID 0 is missing
assertEquals(1, values.nextDoc());
assertEquals(18, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(7, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(7, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(18, values.longValue());
}
assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedIntLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -990,16 +1102,26 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testBasicDouble() throws Exception {
Directory dir = newDirectory();
@ -1082,9 +1204,10 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingDoubleFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
sortField.setMissingValue(Double.NEGATIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1108,19 +1231,28 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(1, values.nextDoc());
assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
} else {
assertEquals(1, values.nextDoc());
assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(2, values.nextDoc());
assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
}
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedDoubleFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
sortField.setMissingValue(Double.NEGATIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1150,21 +1282,32 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingDoubleLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
sortField.setMissingValue(Double.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1188,20 +1331,29 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(1, values.nextDoc());
assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(2, values.nextDoc());
assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
} else {
assertEquals(0, values.nextDoc());
assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(1, values.nextDoc());
assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
}
assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedDoubleLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
sortField.setMissingValue(Double.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1231,16 +1383,26 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testBasicFloat() throws Exception {
Directory dir = newDirectory();
@ -1322,9 +1484,10 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingFloatFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.FLOAT);
SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
sortField.setMissingValue(Float.NEGATIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1348,19 +1511,28 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
assertEquals(1, values.nextDoc());
assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
} else {
assertEquals(1, values.nextDoc());
assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
assertEquals(2, values.nextDoc());
assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
}
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedFloatFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
sortField.setMissingValue(Float.NEGATIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1390,21 +1562,32 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingFloatLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.FLOAT);
SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
sortField.setMissingValue(Float.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1428,20 +1611,29 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(1, values.nextDoc());
assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
assertEquals(2, values.nextDoc());
assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
} else {
assertEquals(0, values.nextDoc());
assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
assertEquals(1, values.nextDoc());
assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
}
assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedFloatLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
sortField.setMissingValue(Float.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1471,16 +1663,26 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testRandom1() throws IOException {
boolean withDeletes = random().nextBoolean();