Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-7869: Changed MemoryIndex to sort 1d points. 

In case of 1d points, the PointInSetQuery.MergePointVisitor expects that these points are visited in ascending order.
Prior to this change the memory index doesn't do this and this can result in document with multiple points that should match to not match.
This commit is contained in:
Martijn van Groningen 2017-06-07 19:55:32 +02:00
parent cf9b70b253
commit 5a34d419b9
No known key found for this signature in database
GPG Key ID: AB236F4FCF2AF12A
3 changed files with 49 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/CHANGES.txt
View File

@ -129,6 +129,10 @@ Bug Fixes
* LUCENE-7864: IndexMergeTool is not using intermediate hard links (even
if possible). (Dawid Weiss)
* LUCENE-7869: Changed MemoryIndex to sort 1d points. In case of 1d points, the PointInSetQuery.MergePointVisitor expects
that these points are visited in ascending order. The memory index doesn't do this and this can result in document
with multiple points that should match to not match. (Martijn van Groningen)
Other
* LUCENE-7800: Remove code that potentially rethrows checked exceptions

35
lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
View File

@ -868,20 +868,27 @@ public class MemoryIndex {
final int numDimensions = fieldInfo.getPointDimensionCount();
final int numBytesPerDimension = fieldInfo.getPointNumBytes();
minPackedValue = pointValues[0].bytes.clone();
maxPackedValue = pointValues[0].bytes.clone();
for (int i = 0; i < pointValuesCount; i++) {
BytesRef pointValue = pointValues[i];
assert pointValue.bytes.length == pointValue.length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this";
for (int dim = 0; dim < numDimensions; ++dim) {
int offset = dim * numBytesPerDimension;
if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, minPackedValue, offset) < 0) {
System.arraycopy(pointValue.bytes, offset, minPackedValue, offset, numBytesPerDimension);
}
if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, maxPackedValue, offset) > 0) {
System.arraycopy(pointValue.bytes, offset, maxPackedValue, offset, numBytesPerDimension);
if (numDimensions == 1) {
// PointInSetQuery.MergePointVisitor expects values to be visited in increasing order,
// this is a 1d optimization which has to be done here too. Otherwise we emit values
// out of order which causes mismatches.
Arrays.sort(pointValues, 0, pointValuesCount);
minPackedValue = pointValues[0].bytes.clone();
maxPackedValue = pointValues[pointValuesCount - 1].bytes.clone();
} else {
minPackedValue = pointValues[0].bytes.clone();
maxPackedValue = pointValues[0].bytes.clone();
for (int i = 0; i < pointValuesCount; i++) {
BytesRef pointValue = pointValues[i];
assert pointValue.bytes.length == pointValue.length : "BytesRef should wrap a precise byte[], BytesRef.deepCopyOf() should take care of this";
for (int dim = 0; dim < numDimensions; ++dim) {
int offset = dim * numBytesPerDimension;
if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, minPackedValue, offset) < 0) {
System.arraycopy(pointValue.bytes, offset, minPackedValue, offset, numBytesPerDimension);
}
if (StringHelper.compare(numBytesPerDimension, pointValue.bytes, offset, maxPackedValue, offset) > 0) {
System.arraycopy(pointValue.bytes, offset, maxPackedValue, offset, numBytesPerDimension);
}
}
}
}

24
lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
View File

@ -512,6 +512,30 @@ public class TestMemoryIndex extends LuceneTestCase {
assertEquals(1, s.count(DoublePoint.newRangeQuery("doubles", new double[] {10D, 10D}, new double[] {30D, 30D})));
}
public void testMultiValuedPointsSortedCorrectly() throws Exception {
Document doc = new Document();
doc.add(new IntPoint("ints", 3));
doc.add(new IntPoint("ints", 2));
doc.add(new IntPoint("ints", 1));
doc.add(new LongPoint("longs", 3L));
doc.add(new LongPoint("longs", 2L));
doc.add(new LongPoint("longs", 1L));
doc.add(new FloatPoint("floats", 3F));
doc.add(new FloatPoint("floats", 2F));
doc.add(new FloatPoint("floats", 1F));
doc.add(new DoublePoint("doubles", 3D));
doc.add(new DoublePoint("doubles", 2D));
doc.add(new DoublePoint("doubles", 1D));
MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
IndexSearcher s = mi.createSearcher();
assertEquals(1, s.count(IntPoint.newSetQuery("ints", 2)));
assertEquals(1, s.count(LongPoint.newSetQuery("longs", 2)));
assertEquals(1, s.count(FloatPoint.newSetQuery("floats", 2)));
assertEquals(1, s.count(DoublePoint.newSetQuery("doubles", 2)));
}
public void testIndexingPointsAndDocValues() throws Exception {
FieldType type = new FieldType();
type.setDimensions(1, 4);