Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Add the ability to set the number of hits to track accurately" 

This reverts commit 84d5015eee.
This commit is contained in:
Jim Ferenczi 2018-12-07 19:15:44 +01:00
parent 84d5015eee
commit da7919f7a4
5 changed files with 705 additions and 941 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lucene/CHANGES.txt
View File

@ -314,14 +314,7 @@ Bug fixes
* LUCENE-8595: Fix interleaved DV update and reset. Interleaved update and reset value
to the same doc in the same updates package looses an update if the reset comes before
the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grand)
* LUCENE-8592: Fix index sorting corruption due to numeric overflow. The merge of sorted segments
can produce an invalid sort if the sort field is an Integer/Long that uses reverse order and contains
values equal to Integer/Long#MIN_VALUE. These values are always sorted first during a merge
(instead of last because of the reverse order) due to this bug. Indices affected by the bug can be
detected by running the CheckIndex command on a distribution that contains the fix (7.6+).
(Jim Ferenczi, Adrien Grand, Mike McCandless, Simon Willnauer)
the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grant)
New Features

27
lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
View File

@ -1718,33 +1718,6 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
}
/**
* Tests that {@link CheckIndex} can detect invalid sort on sorted indices created
* before https://issues.apache.org/jira/browse/LUCENE-8592.
*/
public void testSortedIndexWithInvalidSort() throws Exception {
Path path = createTempDir("sorted");
String name = "sorted-invalid.7.5.0.zip";
InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name);
assertNotNull("Sorted index index " + name + " not found", resource);
TestUtil.unzip(resource, path);
Directory dir = FSDirectory.open(path);
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.leaves().size());
Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
assertNotNull(sort);
assertEquals("<long: \"dateDV\">! missingValue=-9223372036854775808", sort.toString());
reader.close();
CheckIndex.Status status = TestUtil.checkIndex(dir);
assertEquals(1, status.segmentInfos.size());
assertNotNull(status.segmentInfos.get(0).indexSortStatus.error);
assertEquals(status.segmentInfos.get(0).indexSortStatus.error.getMessage(),
"segment has indexSort=<long: \"dateDV\">! missingValue=-9223372036854775808 but docID=4 sorts after docID=5");
dir.close();
}
static long getValue(BinaryDocValues bdv) throws IOException {
BytesRef term = bdv.binaryValue();
int idx = term.offset;

BIN
lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
View File

Binary file not shown.

30
lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
View File

@ -42,18 +42,17 @@ final class MultiSorter {
SortField fields[] = sort.getSort();
final ComparableProvider[][] comparables = new ComparableProvider[fields.length][];
final int[] reverseMuls = new int[fields.length];
for(int i=0;i<fields.length;i++) {
comparables[i] = getComparableProviders(readers, fields[i]);
reverseMuls[i] = fields[i].getReverse() ? -1 : 1;
}
int leafCount = readers.size();
PriorityQueue<LeafAndDocID> queue = new PriorityQueue<LeafAndDocID>(leafCount) {
@Override
public boolean lessThan(LeafAndDocID a, LeafAndDocID b) {
for(int i=0;i<comparables.length;i++) {
int cmp = reverseMuls[i] * a.values[i].compareTo(b.values[i]);
int cmp = a.values[i].compareTo(b.values[i]);
if (cmp != 0) {
return cmp < 0;
}
@ -147,13 +146,14 @@ final class MultiSorter {
/** Returns an object for this docID whose .compareTo represents the requested {@link SortField} sort order. */
private interface ComparableProvider {
Comparable getComparable(int docID) throws IOException;
public Comparable getComparable(int docID) throws IOException;
}
/** Returns {@code ComparableProvider}s for the provided readers to represent the requested {@link SortField} sort order. */
private static ComparableProvider[] getComparableProviders(List<CodecReader> readers, SortField sortField) throws IOException {
ComparableProvider[] providers = new ComparableProvider[readers.size()];
final int reverseMul = sortField.getReverse() ? -1 : 1;
final SortField.Type sortType = Sorter.getSortFieldType(sortField);
switch(sortType) {
@ -169,9 +169,9 @@ final class MultiSorter {
OrdinalMap ordinalMap = OrdinalMap.build(null, values, PackedInts.DEFAULT);
final int missingOrd;
if (sortField.getMissingValue() == SortField.STRING_LAST) {
missingOrd = Integer.MAX_VALUE;
missingOrd = sortField.getReverse() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
} else {
missingOrd = Integer.MIN_VALUE;
missingOrd = sortField.getReverse() ? Integer.MAX_VALUE : Integer.MIN_VALUE;
}
for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
@ -197,7 +197,7 @@ final class MultiSorter {
}
if (readerDocID == docID) {
// translate segment's ord to global ord space:
return Math.toIntExact(globalOrds.get(readerValues.ordValue()));
return reverseMul * (int) globalOrds.get(readerValues.ordValue());
} else {
return missingOrd;
}
@ -238,9 +238,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
return values.longValue();
return reverseMul * values.longValue();
} else {
return missingValue;
return reverseMul * missingValue;
}
}
};
@ -279,9 +279,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
return (int) values.longValue();
return reverseMul * (int) values.longValue();
} else {
return missingValue;
return reverseMul * missingValue;
}
}
};
@ -320,9 +320,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
return Double.longBitsToDouble(values.longValue());
return reverseMul * Double.longBitsToDouble(values.longValue());
} else {
return missingValue;
return reverseMul * missingValue;
}
}
};
@ -361,9 +361,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
return Float.intBitsToFloat((int) values.longValue());
return reverseMul * Float.intBitsToFloat((int) values.longValue());
} else {
return missingValue;
return reverseMul * missingValue;
}
}
};

242
lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
View File

@ -355,10 +355,9 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingStringFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
SortField sortField = new SortField("foo", SortField.Type.STRING);
sortField.setMissingValue(SortField.STRING_FIRST);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -382,29 +381,20 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
SortedDocValues values = leaf.getSortedDocValues("foo");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals("zzz", values.binaryValue().utf8ToString());
assertEquals(1, values.nextDoc());
assertEquals("mmm", values.binaryValue().utf8ToString());
} else {
// docID 0 is missing:
assertEquals(1, values.nextDoc());
assertEquals("mmm", values.binaryValue().utf8ToString());
assertEquals(2, values.nextDoc());
assertEquals("zzz", values.binaryValue().utf8ToString());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedStringFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedSetSortField("foo", reverse);
SortField sortField = new SortedSetSortField("foo", false);
sortField.setMissingValue(SortField.STRING_FIRST);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -435,32 +425,21 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3l, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2l, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1l, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1l, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2l, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3l, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingStringLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
SortField sortField = new SortField("foo", SortField.Type.STRING);
sortField.setMissingValue(SortField.STRING_LAST);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -484,29 +463,20 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
SortedDocValues values = leaf.getSortedDocValues("foo");
if (reverse) {
assertEquals(1, values.nextDoc());
assertEquals("zzz", values.binaryValue().utf8ToString());
assertEquals(2, values.nextDoc());
assertEquals("mmm", values.binaryValue().utf8ToString());
} else {
assertEquals(0, values.nextDoc());
assertEquals("mmm", values.binaryValue().utf8ToString());
assertEquals(1, values.nextDoc());
assertEquals("zzz", values.binaryValue().utf8ToString());
}
assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedStringLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedSetSortField("foo", reverse);
SortField sortField = new SortedSetSortField("foo", false);
sortField.setMissingValue(SortField.STRING_LAST);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -536,26 +506,16 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3l, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2l, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1l, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1l, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2l, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3l, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testBasicLong() throws Exception {
Directory dir = newDirectory();
@ -637,10 +597,9 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingLongFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
SortField sortField = new SortField("foo", SortField.Type.LONG);
sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -664,29 +623,20 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(18, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(7, values.longValue());
} else {
// docID 0 has no value
assertEquals(1, values.nextDoc());
assertEquals(7, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(18, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedLongFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -716,32 +666,21 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingLongLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
SortField sortField = new SortField("foo", SortField.Type.LONG);
sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -765,30 +704,20 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
// docID 0 is missing
assertEquals(1, values.nextDoc());
assertEquals(18, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(7, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(7, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(18, values.longValue());
}
assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedLongLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -819,26 +748,16 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testBasicInt() throws Exception {
Directory dir = newDirectory();
@ -922,10 +841,9 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingIntFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
SortField sortField = new SortField("foo", SortField.Type.INT);
sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -949,28 +867,19 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(18, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(7, values.longValue());
} else {
assertEquals(1, values.nextDoc());
assertEquals(7, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(18, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedIntFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1000,32 +909,21 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingIntLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
SortField sortField = new SortField("foo", SortField.Type.INT);
sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1049,30 +947,20 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
// docID 0 is missing
assertEquals(1, values.nextDoc());
assertEquals(18, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(7, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(7, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(18, values.longValue());
}
assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedIntLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1102,26 +990,16 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testBasicDouble() throws Exception {
Directory dir = newDirectory();
@ -1204,10 +1082,9 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingDoubleFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
sortField.setMissingValue(Double.NEGATIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1231,28 +1108,19 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(1, values.nextDoc());
assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
} else {
assertEquals(1, values.nextDoc());
assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(2, values.nextDoc());
assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
}
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedDoubleFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
sortField.setMissingValue(Double.NEGATIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1282,32 +1150,21 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingDoubleLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
sortField.setMissingValue(Double.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1331,29 +1188,20 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(1, values.nextDoc());
assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(2, values.nextDoc());
assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
} else {
assertEquals(0, values.nextDoc());
assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(1, values.nextDoc());
assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
}
assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedDoubleLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
sortField.setMissingValue(Double.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1383,26 +1231,16 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testBasicFloat() throws Exception {
Directory dir = newDirectory();
@ -1484,10 +1322,9 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingFloatFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
SortField sortField = new SortField("foo", SortField.Type.FLOAT);
sortField.setMissingValue(Float.NEGATIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1511,28 +1348,19 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
assertEquals(1, values.nextDoc());
assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
} else {
assertEquals(1, values.nextDoc());
assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
assertEquals(2, values.nextDoc());
assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
}
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedFloatFirst() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
sortField.setMissingValue(Float.NEGATIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1562,32 +1390,21 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testMissingFloatLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
SortField sortField = new SortField("foo", SortField.Type.FLOAT);
sortField.setMissingValue(Float.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1611,29 +1428,20 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
if (reverse) {
assertEquals(1, values.nextDoc());
assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
assertEquals(2, values.nextDoc());
assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
} else {
assertEquals(0, values.nextDoc());
assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
assertEquals(1, values.nextDoc());
assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
}
assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
}
public void testMissingMultiValuedFloatLast() throws Exception {
for (boolean reverse : new boolean[] {true, false}) {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
sortField.setMissingValue(Float.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
@ -1663,26 +1471,16 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("id");
if (reverse) {
assertEquals(0, values.nextDoc());
assertEquals(3, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(1, values.longValue());
} else {
assertEquals(0, values.nextDoc());
assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
assertEquals(3, values.longValue());
}
r.close();
w.close();
dir.close();
}
}
public void testRandom1() throws IOException {
boolean withDeletes = random().nextBoolean();