Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5399: current state 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene539399@1558451 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-01-15 16:21:31 +00:00
parent 77997d423f
commit 4694ef69eb
20 changed files with 757 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

418
lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
View File

@ -62,6 +62,18 @@ import org.apache.lucene.util.BytesRef;
* <li> {@link #compareBottom} Compare a new hit (docID)
* against the "weakest" (bottom) entry in the queue.
*
* <li> {@link #setTopValue} This method is called by
* {@link TopFieldCollector} to notify the
* FieldComparator of the top most value, which is
* used by future calls to {@link #compareTop}.
*
* <li> {@link #compareBottom} Compare a new hit (docID)
* against the "weakest" (bottom) entry in the queue.
*
* <li> {@link #compareTop} Compare a new hit (docID)
* against the top value previously set by a call to
* {@link #setTopValue}.
*
* <li> {@link #copy} Installs a new hit into the
* priority queue. The {@link FieldValueHitQueue}
* calls this method when a new hit is competitive.
@ -104,7 +116,15 @@ public abstract class FieldComparator<T> {
public abstract void setBottom(final int slot);
/**
* Compare the bottom of the queue with doc. This will
* Record the top value, for future calls to {@link
* #compareTop}. This is only called for searches that
* use searchAfter (deep paging), and is called before any
* calls to {@link #setNextReader}.
*/
public abstract void setTopValue(T value);
/**
* Compare the bottom of the queue with this doc. This will
* only invoked after setBottom has been called. This
* should return the same result as {@link
* #compare(int,int)}} as if bottom were slot1 and the new
@ -122,6 +142,22 @@ public abstract class FieldComparator<T> {
*/
public abstract int compareBottom(int doc) throws IOException;
/**
* Compare the top value with this doc. This will
* only invoked after setTopValue has been called. This
* should return the same result as {@link
* #compare(int,int)}} as if topValue were slot1 and the new
* document were slot 2. This is only called for searches that
* use searchAfter (deep paging).
*
* @param doc that was hit
* @return any N < 0 if the doc's value is sorted after
* the bottom entry (not competitive), any N > 0 if the
* doc's value is sorted before the bottom entry and 0 if
* they are equal.
*/
public abstract int compareTop(int doc) throws IOException;
/**
* This method is called when a new hit is competitive.
* You should copy any state associated with this document
@ -184,10 +220,6 @@ public abstract class FieldComparator<T> {
}
}
/** Returns negative result if the doc's value is less
* than the provided value. */
public abstract int compareDocToValue(int doc, T value) throws IOException;
/**
* Base FieldComparator class for numeric types
*/
@ -223,6 +255,7 @@ public abstract class FieldComparator<T> {
private final DoubleParser parser;
private FieldCache.Doubles currentReaderValues;
private double bottom;
private double topValue;
DoubleComparator(int numHits, String field, FieldCache.Parser parser, Double missingValue) {
super(field, missingValue);
@ -272,21 +305,25 @@ public abstract class FieldComparator<T> {
this.bottom = values[bottom];
}
@Override
public void setTopValue(Double value) {
topValue = value;
}
@Override
public Double value(int slot) {
return Double.valueOf(values[slot]);
}
@Override
public int compareDocToValue(int doc, Double valueObj) {
final double value = valueObj.doubleValue();
public int compareTop(int doc) {
double docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
docValue = missingValue;
}
return Double.compare(docValue, value);
return Double.compare(topValue, docValue);
}
}
@ -297,6 +334,7 @@ public abstract class FieldComparator<T> {
private final FloatParser parser;
private FieldCache.Floats currentReaderValues;
private float bottom;
private float topValue;
FloatComparator(int numHits, String field, FieldCache.Parser parser, Float missingValue) {
super(field, missingValue);
@ -347,21 +385,25 @@ public abstract class FieldComparator<T> {
this.bottom = values[bottom];
}
@Override
public void setTopValue(Float value) {
topValue = value;
}
@Override
public Float value(int slot) {
return Float.valueOf(values[slot]);
}
@Override
public int compareDocToValue(int doc, Float valueObj) {
final float value = valueObj.floatValue();
public int compareTop(int doc) {
float docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
docValue = missingValue;
}
return Float.compare(docValue, value);
return Float.compare(topValue, docValue);
}
}
@ -372,6 +414,7 @@ public abstract class FieldComparator<T> {
private final IntParser parser;
private FieldCache.Ints currentReaderValues;
private int bottom; // Value of bottom of queue
private int topValue;
IntComparator(int numHits, String field, FieldCache.Parser parser, Integer missingValue) {
super(field, missingValue);
@ -421,21 +464,25 @@ public abstract class FieldComparator<T> {
this.bottom = values[bottom];
}
@Override
public void setTopValue(Integer value) {
topValue = value;
}
@Override
public Integer value(int slot) {
return Integer.valueOf(values[slot]);
}
@Override
public int compareDocToValue(int doc, Integer valueObj) {
final int value = valueObj.intValue();
public int compareTop(int doc) {
int docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
docValue = missingValue;
}
return Integer.compare(docValue, value);
return Integer.compare(topValue, docValue);
}
}
@ -446,6 +493,7 @@ public abstract class FieldComparator<T> {
private final LongParser parser;
private FieldCache.Longs currentReaderValues;
private long bottom;
private long topValue;
LongComparator(int numHits, String field, FieldCache.Parser parser, Long missingValue) {
super(field, missingValue);
@ -497,21 +545,25 @@ public abstract class FieldComparator<T> {
this.bottom = values[bottom];
}
@Override
public void setTopValue(Long value) {
topValue = value;
}
@Override
public Long value(int slot) {
return Long.valueOf(values[slot]);
}
@Override
public int compareDocToValue(int doc, Long valueObj) {
final long value = valueObj.longValue();
public int compareTop(int doc) {
long docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
docValue = missingValue;
}
return Long.compare(docValue, value);
return Long.compare(topValue, docValue);
}
}
@ -525,7 +577,8 @@ public abstract class FieldComparator<T> {
private final float[] scores;
private float bottom;
private Scorer scorer;
private float topValue;
RelevanceComparator(int numHits) {
scores = new float[numHits];
}
@ -558,6 +611,11 @@ public abstract class FieldComparator<T> {
this.bottom = scores[bottom];
}
@Override
public void setTopValue(Float value) {
topValue = value;
}
@Override
public void setScorer(Scorer scorer) {
// wrap with a ScoreCachingWrappingScorer so that successive calls to
@ -584,11 +642,10 @@ public abstract class FieldComparator<T> {
}
@Override
public int compareDocToValue(int doc, Float valueObj) throws IOException {
final float value = valueObj.floatValue();
public int compareTop(int doc) throws IOException {
float docValue = scorer.score();
assert !Float.isNaN(docValue);
return Float.compare(value, docValue);
return Float.compare(docValue, topValue);
}
}
@ -597,6 +654,7 @@ public abstract class FieldComparator<T> {
private final int[] docIDs;
private int docBase;
private int bottom;
private int topValue;
DocComparator(int numHits) {
docIDs = new int[numHits];
@ -633,16 +691,20 @@ public abstract class FieldComparator<T> {
this.bottom = docIDs[bottom];
}
@Override
public void setTopValue(Integer value) {
topValue = value;
}
@Override
public Integer value(int slot) {
return Integer.valueOf(docIDs[slot]);
}
@Override
public int compareDocToValue(int doc, Integer valueObj) {
final int value = valueObj.intValue();
public int compareTop(int doc) {
int docValue = docBase + doc;
return Integer.compare(docValue, value);
return Integer.compare(topValue, docValue);
}
}
@ -700,13 +762,42 @@ public abstract class FieldComparator<T> {
@lucene.internal */
BytesRef bottomValue;
/** Set by setTopValue. */
BytesRef topValue;
boolean topSameReader;
int topOrd;
private int docBase;
final BytesRef tempBR = new BytesRef();
/** -1 if missing values are sorted first, 1 if they are
* sorted last */
final int missingSortCmp;
/** Which ordinal to use for a missing value. */
final int missingOrd;
/** Creates this, sorting missing values first. */
public TermOrdValComparator(int numHits, String field) {
this(numHits, field, false);
}
/** Creates this, with control over how missing values
* are sorted. Pass sortMissingLast=true to put
* missing values at the end. */
public TermOrdValComparator(int numHits, String field, boolean sortMissingLast) {
ords = new int[numHits];
values = new BytesRef[numHits];
readerGen = new int[numHits];
this.field = field;
if (sortMissingLast) {
missingSortCmp = 1;
missingOrd = Integer.MAX_VALUE;
} else {
missingSortCmp = -1;
missingOrd = -1;
}
}
@Override
@ -721,140 +812,77 @@ public abstract class FieldComparator<T> {
if (val2 == null) {
return 0;
}
return -1;
return missingSortCmp;
} else if (val2 == null) {
return 1;
return -missingSortCmp;
}
return val1.compareTo(val2);
}
@Override
public int compareBottom(int doc) {
throw new UnsupportedOperationException();
assert bottomSlot != -1;
int docOrd = termsIndex.getOrd(doc);
if (docOrd == -1) {
docOrd = missingOrd;
}
if (bottomSameReader) {
// ord is precisely comparable, even in the equal case
return bottomOrd - docOrd;
} else if (bottomOrd >= docOrd) {
// the equals case always means bottom is > doc
// (because we set bottomOrd to the lower bound in
// setBottom):
return 1;
} else {
return -1;
}
}
@Override
public void copy(int slot, int doc) {
throw new UnsupportedOperationException();
}
@Override
public int compareDocToValue(int doc, BytesRef value) {
int ord = termsIndex.getOrd(doc);
if (ord == -1) {
if (value == null) {
return 0;
ord = missingOrd;
values[slot] = null;
} else {
assert ord >= 0;
if (values[slot] == null) {
values[slot] = new BytesRef();
}
return -1;
} else if (value == null) {
return 1;
termsIndex.lookupOrd(ord, values[slot]);
}
termsIndex.lookupOrd(ord, tempBR);
return tempBR.compareTo(value);
ords[slot] = ord;
readerGen[slot] = currentReaderGen;
}
/** Base class for specialized (per bit width of the
* ords) per-segment comparator. NOTE: this is messy;
* we do this only because hotspot can't reliably inline
* the underlying array access when looking up doc->ord
* @lucene.internal
*/
abstract class PerSegmentComparator extends FieldComparator<BytesRef> {
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
return TermOrdValComparator.this.setNextReader(context);
}
@Override
public int compare(int slot1, int slot2) {
return TermOrdValComparator.this.compare(slot1, slot2);
}
@Override
public void setBottom(final int bottom) {
TermOrdValComparator.this.setBottom(bottom);
}
@Override
public BytesRef value(int slot) {
return TermOrdValComparator.this.value(slot);
}
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
if (val1 == null) {
if (val2 == null) {
return 0;
}
return -1;
} else if (val2 == null) {
return 1;
}
return val1.compareTo(val2);
}
@Override
public int compareDocToValue(int doc, BytesRef value) {
return TermOrdValComparator.this.compareDocToValue(doc, value);
}
}
// Used per-segment when docToOrd is null:
private final class AnyOrdComparator extends PerSegmentComparator {
private final SortedDocValues termsIndex;
private final int docBase;
public AnyOrdComparator(SortedDocValues termsIndex, int docBase) {
this.termsIndex = termsIndex;
this.docBase = docBase;
}
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
final int docOrd = termsIndex.getOrd(doc);
if (bottomSameReader) {
// ord is precisely comparable, even in the equal case
return bottomOrd - docOrd;
} else if (bottomOrd >= docOrd) {
// the equals case always means bottom is > doc
// (because we set bottomOrd to the lower bound in
// setBottom):
return 1;
} else {
return -1;
}
}
@Override
public void copy(int slot, int doc) {
final int ord = termsIndex.getOrd(doc);
ords[slot] = ord;
if (ord == -1) {
values[slot] = null;
} else {
assert ord >= 0;
if (values[slot] == null) {
values[slot] = new BytesRef();
}
termsIndex.lookupOrd(ord, values[slot]);
}
readerGen[slot] = currentReaderGen;
}
}
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
final int docBase = context.docBase;
docBase = context.docBase;
termsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), field);
FieldComparator<BytesRef> perSegComp = new AnyOrdComparator(termsIndex, docBase);
currentReaderGen++;
if (bottomSlot != -1) {
perSegComp.setBottom(bottomSlot);
if (topValue != null) {
// Recompute topOrd/SameReader
int ord = termsIndex.lookupTerm(topValue);
if (ord >= 0) {
topSameReader = true;
topOrd = ord;
} else {
topSameReader = false;
topOrd = -ord-2;
}
} else {
topOrd = missingOrd;
topSameReader = true;
}
return perSegComp;
if (bottomSlot != -1) {
// Recompute bottomOrd/SameReader
setBottom(bottomSlot);
}
return this;
}
@Override
@ -867,18 +895,18 @@ public abstract class FieldComparator<T> {
bottomSameReader = true;
} else {
if (bottomValue == null) {
// -1 ord is null for all segments
assert ords[bottomSlot] == -1;
bottomOrd = -1;
// missingOrd is null for all segments
assert ords[bottomSlot] == missingOrd;
bottomOrd = missingOrd;
bottomSameReader = true;
readerGen[bottomSlot] = currentReaderGen;
} else {
final int index = termsIndex.lookupTerm(bottomValue);
if (index < 0) {
bottomOrd = -index - 2;
final int ord = termsIndex.lookupTerm(bottomValue);
if (ord < 0) {
bottomOrd = -ord - 2;
bottomSameReader = false;
} else {
bottomOrd = index;
bottomOrd = ord;
// exact value match
bottomSameReader = true;
readerGen[bottomSlot] = currentReaderGen;
@ -888,28 +916,71 @@ public abstract class FieldComparator<T> {
}
}
@Override
public void setTopValue(BytesRef value) {
// null is fine: it means the last doc of the prior
// search was missing this value
topValue = value;
}
@Override
public BytesRef value(int slot) {
return values[slot];
}
@Override
public int compareTop(int doc) {
int ord = termsIndex.getOrd(doc);
if (topSameReader) {
// ord is precisely comparable, even in the equal case
return topOrd - ord;
} else if (ord <= topOrd) {
// the equals case always means doc is < value
// (because we set lastOrd to the lower bound)
return 1;
} else {
return -1;
}
}
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
if (val1 == null) {
if (val2 == null) {
return 0;
}
return missingSortCmp;
} else if (val2 == null) {
return -missingSortCmp;
}
return val1.compareTo(val2);
}
}
// just used internally in this comparator
private static final byte[] MISSING_BYTES = new byte[0];
/** Sorts by field's natural Term sort order. All
* comparisons are done using BytesRef.compareTo, which is
* slow for medium to large result sets but possibly
* very fast for very small results sets. */
// TODO: should we remove this? who really uses it?
public static final class TermValComparator extends FieldComparator<BytesRef> {
// sentinels, just used internally in this comparator
private static final byte[] MISSING_BYTES = new byte[0];
private static final byte[] NON_MISSING_BYTES = new byte[0];
private BytesRef[] values;
private BinaryDocValues docTerms;
private Bits docsWithField;
private final String field;
private BytesRef bottom;
private BytesRef topValue;
private final BytesRef tempBR = new BytesRef();
// TODO: add missing first/last support here?
/** Sole constructor. */
TermValComparator(int numHits, String field) {
values = new BytesRef[numHits];
this.field = field;
@ -919,12 +990,12 @@ public abstract class FieldComparator<T> {
public int compare(int slot1, int slot2) {
final BytesRef val1 = values[slot1];
final BytesRef val2 = values[slot2];
if (val1 == null) {
if (val2 == null) {
if (val1.bytes == MISSING_BYTES) {
if (val2.bytes == MISSING_BYTES) {
return 0;
}
return -1;
} else if (val2 == null) {
} else if (val2.bytes == MISSING_BYTES) {
return 1;
}
@ -934,18 +1005,8 @@ public abstract class FieldComparator<T> {
@Override
public int compareBottom(int doc) {
docTerms.get(doc, tempBR);
if (tempBR.length == 0 && docsWithField.get(doc) == false) {
tempBR.bytes = MISSING_BYTES;
}
if (bottom.bytes == MISSING_BYTES) {
if (tempBR.bytes == MISSING_BYTES) {
return 0;
}
return -1;
} else if (tempBR.bytes == MISSING_BYTES) {
return 1;
}
return bottom.compareTo(tempBR);
setMissingBytes(doc, tempBR);
return compareValues(bottom, tempBR);
}
@Override
@ -954,9 +1015,7 @@ public abstract class FieldComparator<T> {
values[slot] = new BytesRef();
}
docTerms.get(doc, values[slot]);
if (values[slot].length == 0 && docsWithField.get(doc) == false) {
values[slot].bytes = MISSING_BYTES;
}
setMissingBytes(doc, values[slot]);
}
@Override
@ -971,6 +1030,14 @@ public abstract class FieldComparator<T> {
this.bottom = values[bottom];
}
@Override
public void setTopValue(BytesRef value) {
if (value == null) {
throw new IllegalArgumentException("value cannot be null");
}
topValue = value;
}
@Override
public BytesRef value(int slot) {
return values[slot];
@ -978,24 +1045,33 @@ public abstract class FieldComparator<T> {
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
if (val1 == null) {
if (val2 == null) {
// missing always sorts first:
if (val1.bytes == MISSING_BYTES) {
if (val2.bytes == MISSING_BYTES) {
return 0;
}
return -1;
} else if (val2 == null) {
} else if (val2.bytes == MISSING_BYTES) {
return 1;
}
return val1.compareTo(val2);
}
@Override
public int compareDocToValue(int doc, BytesRef value) {
public int compareTop(int doc) {
docTerms.get(doc, tempBR);
if (tempBR.length == 0 && docsWithField.get(doc) == false) {
tempBR.bytes = MISSING_BYTES;
setMissingBytes(doc, tempBR);
return compareValues(topValue, tempBR);
}
private void setMissingBytes(int doc, BytesRef br) {
if (br.length == 0) {
if (docsWithField.get(doc) == false) {
br.bytes = MISSING_BYTES;
} else {
br.bytes = NON_MISSING_BYTES;
}
}
return tempBR.compareTo(value);
}
}
}

12
lucene/core/src/java/org/apache/lucene/search/FieldDoc.java
View File

@ -17,6 +17,8 @@ package org.apache.lucene.search;
* limitations under the License.
*/
import java.util.Arrays;
/**
* Expert: A ScoreDoc which also contains information about
* how to sort the referenced document. In addition to the
@ -69,14 +71,10 @@ public class FieldDoc extends ScoreDoc {
@Override
public String toString() {
// super.toString returns the doc and score information, so just add the
// fields information
// fields information
StringBuilder sb = new StringBuilder(super.toString());
sb.append("[");
for (int i = 0; i < fields.length; i++) {
sb.append(fields[i]).append(", ");
}
sb.setLength(sb.length() - 2); // discard last ", "
sb.append("]");
sb.append(" fields=");
sb.append(Arrays.toString(fields));
return sb.toString();
}
}

25
lucene/core/src/java/org/apache/lucene/search/SortField.java
View File

@ -106,6 +106,9 @@ public class SortField {
// Used for 'sortMissingFirst/Last'
public Object missingValue = null;
// Only used with type=STRING
public boolean sortMissingLast;
/** Creates a sort by terms in the given field with the type of term
* values explicitly given.
* @param field Name of field to sort by. Can be <code>null</code> if
@ -165,13 +168,24 @@ public class SortField {
this.reverse = reverse;
this.parser = parser;
}
/** Pass this to {@link #setMissingValue} to have missing
* string values sort first. */
public final static Object STRING_FIRST = new Object();
public SortField setMissingValue(Object missingValue) {
if (type != Type.INT && type != Type.FLOAT && type != Type.LONG && type != Type.DOUBLE) {
throw new IllegalArgumentException( "Missing value only works for numeric types" );
/** Pass this to {@link #setMissingValue} to have missing
* string values sort last. */
public final static Object STRING_LAST = new Object();
public void setMissingValue(Object missingValue) {
if (type == Type.STRING) {
if (missingValue != STRING_FIRST && missingValue != STRING_LAST) {
throw new IllegalArgumentException("For STRING type, missing value must be either STRING_FIRST or STRING_LAST");
}
} else if (type != Type.INT && type != Type.FLOAT && type != Type.LONG && type != Type.DOUBLE) {
throw new IllegalArgumentException("Missing value only works for numeric or STRING types");
}
this.missingValue = missingValue;
return this;
}
/** Creates a sort with a custom comparison function.
@ -376,9 +390,10 @@ public class SortField {
return comparatorSource.newComparator(field, numHits, sortPos, reverse);
case STRING:
return new FieldComparator.TermOrdValComparator(numHits, field);
return new FieldComparator.TermOrdValComparator(numHits, field, missingValue == STRING_LAST);
case STRING_VAL:
// TODO: should we remove this? who really uses it?
return new FieldComparator.TermValComparator(numHits, field);
case REWRITEABLE:

72
lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java
View File

@ -869,6 +869,13 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
// Must set maxScore to NEG_INF, or otherwise Math.max always returns NaN.
maxScore = Float.NEGATIVE_INFINITY;
// Tell all comparators their top value:
for(int i=0;i<comparators.length;i++) {
@SuppressWarnings("unchecked")
FieldComparator<Object> comparator = (FieldComparator<Object>) comparators[i];
comparator.setTopValue(after.fields[i]);
}
}
void updateBottom(int doc, float score) {
@ -882,36 +889,6 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
public void collect(int doc) throws IOException {
totalHits++;
//System.out.println(" collect doc=" + doc);
// Check if this hit was already collected on a
// previous page:
boolean sameValues = true;
for(int compIDX=0;compIDX<comparators.length;compIDX++) {
final FieldComparator comp = comparators[compIDX];
final int cmp = reverseMul[compIDX] * comp.compareDocToValue(doc, after.fields[compIDX]);
if (cmp < 0) {
// Already collected on a previous page
//System.out.println(" skip: before");
return;
} else if (cmp > 0) {
// Not yet collected
sameValues = false;
//System.out.println(" keep: after");
break;
}
}
// Tie-break by docID:
if (sameValues && doc <= afterDoc) {
// Already collected on a previous page
//System.out.println(" skip: tie-break");
return;
}
collectedHits++;
float score = Float.NaN;
if (trackMaxScore) {
score = scorer.score();
@ -921,7 +898,8 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
}
if (queueFull) {
// Fastmatch: return if this hit is not competitive
// Fastmatch: return if this hit is no better than
// the worst hit currently in the queue:
for (int i = 0;; i++) {
final int c = reverseMul[i] * comparators[i].compareBottom(doc);
if (c < 0) {
@ -939,7 +917,37 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
break;
}
}
}
//System.out.println(" collect doc=" + doc);
// Check if this hit was already collected on a
// previous page:
boolean sameValues = true;
for(int compIDX=0;compIDX<comparators.length;compIDX++) {
final FieldComparator comp = comparators[compIDX];
final int cmp = reverseMul[compIDX] * comp.compareTop(doc);
if (cmp > 0) {
// Already collected on a previous page
//System.out.println(" skip: before");
return;
} else if (cmp < 0) {
// Not yet collected
sameValues = false;
//System.out.println(" keep: after");
break;
}
}
// Tie-break by docID:
if (sameValues && doc <= afterDoc) {
// Already collected on a previous page
//System.out.println(" skip: tie-break");
return;
}
if (queueFull) {
// This hit is competitive - replace bottom element in queue & adjustTop
for (int i = 0; i < comparators.length; i++) {
comparators[i].copy(bottom.slot, doc);
@ -955,6 +963,8 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
comparators[i].setBottom(bottom.slot);
}
} else {
collectedHits++;
// Startup transient: queue hasn't gathered numHits yet
final int slot = collectedHits - 1;
//System.out.println(" slot=" + slot);

2
lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java
View File

@ -96,7 +96,7 @@ package org.apache.lucene.util;
public final class UnicodeUtil {
/** A binary term consisting of a number of 0xff bytes, likely to be bigger than other terms
* one would normally encounter, and definitely bigger than any UTF-8 terms.
* (e.g. collation keys) one would normally encounter, and definitely bigger than any UTF-8 terms.
* <p>
* WARNING: This is not a valid UTF8 Term
**/

7
lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
View File

@ -145,6 +145,11 @@ final class JustCompileSearch {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
@Override
public void setTopValue(Object value) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
@Override
public FieldComparator<Object> setNextReader(AtomicReaderContext context) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
@ -156,7 +161,7 @@ final class JustCompileSearch {
}
@Override
public int compareDocToValue(int doc, Object value) {
public int compareTop(int doc) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
}

12
lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java
View File

@ -157,6 +157,11 @@ class ElevationComparatorSource extends FieldComparatorSource {
bottomVal = values[slot];
}
@Override
public void setTopValue(Integer value) {
throw new UnsupportedOperationException();
}
private int docVal(int doc) {
int ord = idIndex.getOrd(doc);
if (ord == -1) {
@ -190,11 +195,8 @@ class ElevationComparatorSource extends FieldComparatorSource {
}
@Override
public int compareDocToValue(int doc, Integer valueObj) {
final int value = valueObj.intValue();
final int docValue = docVal(doc);
// values will be small enough that there is no overflow concern
return value - docValue;
public int compareTop(int doc) {
throw new UnsupportedOperationException();
}
};
}

139
lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java
View File

@ -17,7 +17,10 @@ package org.apache.lucene.search;
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
@ -29,6 +32,7 @@ import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@ -45,6 +49,9 @@ public class TestSearchAfter extends LuceneTestCase {
private Directory dir;
private IndexReader reader;
private IndexSearcher searcher;
private int iter;
// nocommit add SF.setMissingValue too
@Override
public void setUp() throws Exception {
@ -53,31 +60,51 @@ public class TestSearchAfter extends LuceneTestCase {
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
int numDocs = atLeast(200);
for (int i = 0; i < numDocs; i++) {
List<Field> fields = new ArrayList<Field>();
fields.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
fields.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
fields.add(newStringField("byte", "" + ((byte) random().nextInt()), Field.Store.NO));
fields.add(newStringField("short", "" + ((short) random().nextInt()), Field.Store.NO));
fields.add(new IntField("int", random().nextInt(), Field.Store.NO));
fields.add(new LongField("long", random().nextLong(), Field.Store.NO));
fields.add(new FloatField("float", random().nextFloat(), Field.Store.NO));
fields.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
fields.add(newStringField("bytes", _TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
fields.add(newStringField("bytesval", _TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
fields.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
fields.add(new NumericDocValuesField("intdocvalues", random().nextInt()));
fields.add(new FloatDocValuesField("floatdocvalues", random().nextFloat()));
fields.add(new SortedDocValuesField("sortedbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
fields.add(new SortedDocValuesField("sortedbytesdocvaluesval", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
fields.add(new BinaryDocValuesField("straightbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
Document document = new Document();
document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
document.add(newStringField("byte", "" + ((byte) random().nextInt()), Field.Store.NO));
document.add(newStringField("short", "" + ((short) random().nextInt()), Field.Store.NO));
document.add(new IntField("int", random().nextInt(), Field.Store.NO));
document.add(new LongField("long", random().nextLong(), Field.Store.NO));
document.add(new FloatField("float", random().nextFloat(), Field.Store.NO));
document.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
document.add(newStringField("bytes", _TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
document.add(newStringField("bytesval", _TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
document.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
document.add(new NumericDocValuesField("intdocvalues", random().nextInt()));
document.add(new FloatDocValuesField("floatdocvalues", random().nextFloat()));
document.add(new SortedDocValuesField("sortedbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
document.add(new SortedDocValuesField("sortedbytesdocvaluesval", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
document.add(new BinaryDocValuesField("straightbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
document.add(new StoredField("id", ""+i));
if (VERBOSE) {
System.out.println(" add doc id=" + i);
}
for(Field field : fields) {
// So we are sometimes missing that field:
if (random().nextInt(5) != 4) {
document.add(field);
if (VERBOSE) {
System.out.println(" " + field);
}
}
}
iw.addDocument(document);
// nocommit randomly commit so we exercise segments
}
reader = iw.getReader();
iw.close();
searcher = newSearcher(reader);
if (VERBOSE) {
System.out.println(" searcher=" + searcher);
}
}
@Override
@ -104,25 +131,53 @@ public class TestSearchAfter extends LuceneTestCase {
assertQuery(bq, null);
}
}
static SortField[] allSortFields = new SortField[] {
new SortField("int", SortField.Type.INT, false),
new SortField("long", SortField.Type.LONG, false),
new SortField("float", SortField.Type.FLOAT, false),
new SortField("double", SortField.Type.DOUBLE, false),
new SortField("bytes", SortField.Type.STRING, false),
new SortField("bytesval", SortField.Type.STRING_VAL, false),
new SortField("intdocvalues", SortField.Type.INT, false),
new SortField("floatdocvalues", SortField.Type.FLOAT, false),
new SortField("sortedbytesdocvalues", SortField.Type.STRING, false),
new SortField("sortedbytesdocvaluesval", SortField.Type.STRING_VAL, false),
new SortField("straightbytesdocvalues", SortField.Type.STRING_VAL, false),
new SortField("int", SortField.Type.INT, true),
new SortField("long", SortField.Type.LONG, true),
new SortField("float", SortField.Type.FLOAT, true),
new SortField("double", SortField.Type.DOUBLE, true),
new SortField("bytes", SortField.Type.STRING, true),
new SortField("bytesval", SortField.Type.STRING_VAL, true),
new SortField("intdocvalues", SortField.Type.INT, true),
new SortField("floatdocvalues", SortField.Type.FLOAT, true),
new SortField("sortedbytesdocvalues", SortField.Type.STRING, true),
new SortField("sortedbytesdocvaluesval", SortField.Type.STRING_VAL, true),
new SortField("straightbytesdocvalues", SortField.Type.STRING_VAL, true),
SortField.FIELD_SCORE,
SortField.FIELD_DOC,
// nocommit add RELEVANCE too
};
void assertQuery(Query query, Filter filter) throws Exception {
assertQuery(query, filter, null);
assertQuery(query, filter, Sort.RELEVANCE);
assertQuery(query, filter, Sort.INDEXORDER);
for(int rev=0;rev<2;rev++) {
boolean reversed = rev == 1;
assertQuery(query, filter, new Sort(new SortField[] {new SortField("int", SortField.Type.INT, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("long", SortField.Type.LONG, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("float", SortField.Type.FLOAT, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("double", SortField.Type.DOUBLE, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("bytes", SortField.Type.STRING, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("bytesval", SortField.Type.STRING_VAL, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("intdocvalues", SortField.Type.INT, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("floatdocvalues", SortField.Type.FLOAT, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("sortedbytesdocvalues", SortField.Type.STRING, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("sortedbytesdocvaluesval", SortField.Type.STRING_VAL, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("straightbytesdocvalues", SortField.Type.STRING_VAL, reversed)}));
for(SortField sortField : allSortFields) {
assertQuery(query, filter, new Sort(new SortField[] {sortField}));
}
for(int i=0;i<20;i++) {
assertQuery(query, filter, getRandomSort());
}
}
Sort getRandomSort() {
SortField[] sortFields = new SortField[_TestUtil.nextInt(random(), 2, 7)];
for(int i=0;i<sortFields.length;i++) {
sortFields[i] = allSortFields[random().nextInt(allSortFields.length)];
}
return new Sort(sortFields);
}
void assertQuery(Query query, Filter filter, Sort sort) throws Exception {
@ -130,18 +185,24 @@ public class TestSearchAfter extends LuceneTestCase {
TopDocs all;
int pageSize = _TestUtil.nextInt(random(), 1, maxDoc*2);
if (VERBOSE) {
System.out.println("\nassertQuery: query=" + query + " filter=" + filter + " sort=" + sort + " pageSize=" + pageSize);
System.out.println("\nassertQuery " + (iter++) + ": query=" + query + " filter=" + filter + " sort=" + sort + " pageSize=" + pageSize);
}
final boolean doMaxScore = random().nextBoolean();
if (sort == null) {
// nocommit randomly sometimes doScores & doMaxScore:
all = searcher.search(query, filter, maxDoc);
} else if (sort == Sort.RELEVANCE) {
all = searcher.search(query, filter, maxDoc, sort, true, doMaxScore);
} else {
// nocommit randomly sometimes doScores & doMaxScore:
all = searcher.search(query, filter, maxDoc, sort);
}
if (VERBOSE) {
System.out.println(" all.totalHits=" + all.totalHits);
int upto = 0;
for(ScoreDoc scoreDoc : all.scoreDocs) {
System.out.println(" hit " + (upto++) + ": id=" + searcher.doc(scoreDoc.doc).get("id") + " " + scoreDoc);
}
}
int pageStart = 0;
ScoreDoc lastBottom = null;
@ -151,17 +212,22 @@ public class TestSearchAfter extends LuceneTestCase {
if (VERBOSE) {
System.out.println(" iter lastBottom=" + lastBottom);
}
// nocommit randomly sometimes doScores & doMaxScore:
paged = searcher.searchAfter(lastBottom, query, filter, pageSize);
} else {
if (VERBOSE) {
System.out.println(" iter lastBottom=" + lastBottom + (lastBottom == null ? "" : " fields=" + Arrays.toString(((FieldDoc) lastBottom).fields)));
System.out.println(" iter lastBottom=" + lastBottom);
}
if (sort == Sort.RELEVANCE) {
paged = searcher.searchAfter(lastBottom, query, filter, pageSize, sort, true, doMaxScore);
} else {
// nocommit randomly sometimes doScores & doMaxScore:
paged = searcher.searchAfter(lastBottom, query, filter, pageSize, sort);
}
}
if (VERBOSE) {
System.out.println(" " + paged.scoreDocs.length + " hits on page");
}
if (paged.scoreDocs.length == 0) {
break;
@ -173,11 +239,16 @@ public class TestSearchAfter extends LuceneTestCase {
assertEquals(all.scoreDocs.length, pageStart);
}
static void assertPage(int pageStart, TopDocs all, TopDocs paged) {
void assertPage(int pageStart, TopDocs all, TopDocs paged) throws IOException {
assertEquals(all.totalHits, paged.totalHits);
for (int i = 0; i < paged.scoreDocs.length; i++) {
ScoreDoc sd1 = all.scoreDocs[pageStart + i];
ScoreDoc sd2 = paged.scoreDocs[i];
if (VERBOSE) {
System.out.println(" hit " + (pageStart + i));
System.out.println(" expected id=" + searcher.doc(sd1.doc).get("id") + " " + sd1);
System.out.println(" actual id=" + searcher.doc(sd2.doc).get("id") + " " + sd2);
}
assertEquals(sd1.doc, sd2.doc);
assertEquals(sd1.score, sd2.score, 0f);
if (sd1 instanceof FieldDoc) {

126
lucene/core/src/test/org/apache/lucene/search/TestSort.java
View File

@ -196,6 +196,132 @@ public class TestSort extends LuceneTestCase {
ir.close();
dir.close();
}
/** Tests sorting on type string with a missing
* value sorted first */
public void testStringMissingSortedFirst() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "foo", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "bar", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sf = new SortField("value", SortField.Type.STRING);
Sort sort = new Sort(sf);
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// null comes first
assertNull(searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("foo", searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests reverse sorting on type string with a missing
* value sorted first */
public void testStringMissingSortedFirstReverse() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "foo", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "bar", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sf = new SortField("value", SortField.Type.STRING, true);
Sort sort = new Sort(sf);
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
assertEquals("foo", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value"));
// null comes last
assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests sorting on type string with a missing
* value sorted last */
public void testStringValMissingSortedLast() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "foo", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "bar", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sf = new SortField("value", SortField.Type.STRING);
sf.setMissingValue(SortField.STRING_LAST);
Sort sort = new Sort(sf);
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value"));
// null comes last
assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests reverse sorting on type string with a missing
* value sorted last */
public void testStringValMissingSortedLastReverse() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "foo", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "bar", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sf = new SortField("value", SortField.Type.STRING, true);
sf.setMissingValue(SortField.STRING_LAST);
Sort sort = new Sort(sf);
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// null comes first
assertNull(searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("bar", searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests reverse sorting on type string_val */
public void testStringValReverse() throws IOException {

128
lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
@ -29,6 +30,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
@ -60,30 +62,44 @@ public class TestSortRandom extends LuceneTestCase {
final List<BytesRef> docValues = new ArrayList<BytesRef>();
// TODO: deletions
while (numDocs < NUM_DOCS) {
final String s;
if (random.nextBoolean()) {
s = _TestUtil.randomSimpleString(random, maxLength);
} else {
s = _TestUtil.randomUnicodeString(random, maxLength);
}
final BytesRef br = new BytesRef(s);
if (!allowDups) {
if (seen.contains(s)) {
continue;
}
seen.add(s);
}
if (VERBOSE) {
System.out.println(" " + numDocs + ": s=" + s);
}
final Document doc = new Document();
doc.add(new SortedDocValuesField("stringdv", br));
doc.add(newStringField("string", s, Field.Store.NO));
// 10% of the time, the document is missing the value:
final BytesRef br;
if (random().nextInt(10) != 7) {
final String s;
if (random.nextBoolean()) {
s = _TestUtil.randomSimpleString(random, maxLength);
} else {
s = _TestUtil.randomUnicodeString(random, maxLength);
}
if (!allowDups) {
if (seen.contains(s)) {
continue;
}
seen.add(s);
}
if (VERBOSE) {
System.out.println(" " + numDocs + ": s=" + s);
}
br = new BytesRef(s);
doc.add(new SortedDocValuesField("stringdv", br));
doc.add(newStringField("string", s, Field.Store.NO));
docValues.add(br);
} else {
br = null;
if (VERBOSE) {
System.out.println(" " + numDocs + ": <missing>");
}
docValues.add(null);
}
doc.add(new NumericDocValuesField("id", numDocs));
docValues.add(br);
doc.add(new StoredField("id", numDocs));
writer.addDocument(doc);
numDocs++;
@ -103,13 +119,26 @@ public class TestSortRandom extends LuceneTestCase {
final int ITERS = atLeast(100);
for(int iter=0;iter<ITERS;iter++) {
final boolean reverse = random.nextBoolean();
final TopFieldDocs hits;
final SortField sf;
final boolean sortMissingLast;
final boolean missingIsNull;
if (random.nextBoolean()) {
sf = new SortField("stringdv", SortField.Type.STRING, reverse);
// Can only use sort missing if the DVFormat
// supports docsWithField:
sortMissingLast = defaultCodecSupportsDocsWithField() && random().nextBoolean();
missingIsNull = defaultCodecSupportsDocsWithField();
} else {
sf = new SortField("string", SortField.Type.STRING, reverse);
sortMissingLast = random().nextBoolean();
missingIsNull = true;
}
if (sortMissingLast) {
sf.setMissingValue(SortField.STRING_LAST);
}
final Sort sort;
if (random.nextBoolean()) {
sort = new Sort(sf);
@ -138,11 +167,34 @@ public class TestSortRandom extends LuceneTestCase {
}
if (VERBOSE) {
System.out.println("\nTEST: iter=" + iter + " " + hits.totalHits + " hits; topN=" + hitCount + "; reverse=" + reverse);
System.out.println("\nTEST: iter=" + iter + " " + hits.totalHits + " hits; topN=" + hitCount + "; reverse=" + reverse + "; sortMissingLast=" + sortMissingLast + " sort=" + sort);
}
// Compute expected results:
Collections.sort(f.matchValues);
Collections.sort(f.matchValues, new Comparator<BytesRef>() {
@Override
public int compare(BytesRef a, BytesRef b) {
if (a == null) {
if (b == null) {
return 0;
}
if (sortMissingLast) {
return 1;
} else {
return -1;
}
} else if (b == null) {
if (sortMissingLast) {
return -1;
} else {
return 1;
}
} else {
return a.compareTo(b);
}
}
});
if (reverse) {
Collections.reverse(f.matchValues);
}
@ -150,7 +202,11 @@ public class TestSortRandom extends LuceneTestCase {
if (VERBOSE) {
System.out.println(" expected:");
for(int idx=0;idx<expected.size();idx++) {
System.out.println(" " + idx + ": " + expected.get(idx).utf8ToString());
BytesRef br = expected.get(idx);
if (br == null && missingIsNull == false) {
br = new BytesRef();
}
System.out.println(" " + idx + ": " + (br == null ? "<missing>" : br.utf8ToString()));
if (idx == hitCount-1) {
break;
}
@ -161,12 +217,30 @@ public class TestSortRandom extends LuceneTestCase {
System.out.println(" actual:");
for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
System.out.println(" " + hitIDX + ": " + ((BytesRef) fd.fields[0]).utf8ToString());
BytesRef br = (BytesRef) fd.fields[0];
System.out.println(" " + hitIDX + ": " + (br == null ? "<missing>" : br.utf8ToString()) + " id=" + s.doc(fd.doc).get("id"));
}
}
for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
assertEquals(expected.get(hitIDX), (BytesRef) fd.fields[0]);
BytesRef br = expected.get(hitIDX);
if (br == null && missingIsNull == false) {
br = new BytesRef();
}
// Normally, the old codecs (that don't support
// docsWithField via doc values) will always return
// an empty BytesRef for the missing case; however,
// if all docs in a given segment were missing, in
// that case it will return null! So we must map
// null here, too:
BytesRef br2 = (BytesRef) fd.fields[0];
if (br2 == null && missingIsNull == false) {
br2 = new BytesRef();
}
assertEquals(br, br2);
}
}

10
lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionComparator.java
View File

@ -30,6 +30,7 @@ import org.apache.lucene.search.Scorer;
class ExpressionComparator extends FieldComparator<Double> {
private final double[] values;
private double bottom;
private double topValue;
private ValueSource source;
private FunctionValues scores;
@ -66,6 +67,11 @@ class ExpressionComparator extends FieldComparator<Double> {
bottom = values[slot];
}
@Override
public void setTopValue(Double value) {
topValue = value.doubleValue();
}
@Override
public int compareBottom(int doc) throws IOException {
return Double.compare(bottom, scores.doubleVal(doc));
@ -88,7 +94,7 @@ class ExpressionComparator extends FieldComparator<Double> {
}
@Override
public int compareDocToValue(int doc, Double valueObj) throws IOException {
return Double.compare(scores.doubleVal(doc), valueObj.doubleValue());
public int compareTop(int doc) throws IOException {
return Double.compare(topValue, scores.doubleVal(doc));
}
}

13
lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinFieldComparator.java
View File

@ -59,6 +59,11 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
wrappedComparator.setBottom(slot);
}
@Override
public void setTopValue(Object value) {
wrappedComparator.setTopValue(value);
}
@Override
public FieldComparator<Object> setNextReader(AtomicReaderContext context) throws IOException {
DocIdSet innerDocuments = childFilter.getDocIdSet(context, null);
@ -193,7 +198,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
@Override
@SuppressWarnings("unchecked")
public int compareDocToValue(int parentDoc, Object value) throws IOException {
public int compareTop(int parentDoc) throws IOException {
if (parentDoc == 0 || parentDocuments == null || childDocuments == null) {
return 0;
}
@ -216,7 +221,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
if (childDoc >= parentDoc || childDoc == -1) {
return cmp;
}
int cmp1 = wrappedComparator.compareDocToValue(childDoc, value);
int cmp1 = wrappedComparator.compareTop(childDoc);
if (cmp1 > 0) {
return cmp1;
} else {
@ -309,7 +314,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
@Override
@SuppressWarnings("unchecked")
public int compareDocToValue(int parentDoc, Object value) throws IOException {
public int compareTop(int parentDoc) throws IOException {
if (parentDoc == 0 || parentDocuments == null || childDocuments == null) {
return 0;
}
@ -330,7 +335,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
if (childDoc >= parentDoc || childDoc == -1) {
return cmp;
}
int cmp1 = wrappedComparator.compareDocToValue(childDoc, value);
int cmp1 = wrappedComparator.compareTop(childDoc);
if (cmp1 < 0) {
return cmp1;
} else {

11
lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
View File

@ -131,6 +131,7 @@ public abstract class ValueSource {
private FunctionValues docVals;
private double bottom;
private final Map fcontext;
private double topValue;
ValueSourceComparator(Map fcontext, int numHits) {
this.fcontext = fcontext;
@ -163,16 +164,20 @@ public abstract class ValueSource {
this.bottom = values[bottom];
}
@Override
public void setTopValue(final Double value) {
this.topValue = value.doubleValue();
}
@Override
public Double value(int slot) {
return values[slot];
}
@Override
public int compareDocToValue(int doc, Double valueObj) {
final double value = valueObj;
public int compareTop(int doc) {
final double docValue = docVals.doubleVal(doc);
return Double.compare(docValue, value);
return Double.compare(topValue, docValue);
}
}
}

10
lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java
View File

@ -44,6 +44,7 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
private final String field;
final Collator collator;
private String bottom;
private String topValue;
private final BytesRef tempBR = new BytesRef();
public SlowCollatedStringComparator(int numHits, String field, Collator collator) {
@ -104,6 +105,11 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
this.bottom = values[bottom];
}
@Override
public void setTopValue(final String value) {
this.topValue = value;
}
@Override
public String value(int slot) {
return values[slot];
@ -124,7 +130,7 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
}
@Override
public int compareDocToValue(int doc, String value) {
public int compareTop(int doc) {
currentDocTerms.get(doc, tempBR);
final String docValue;
if (tempBR.length == 0 && docsWithField.get(doc) == false) {
@ -132,6 +138,6 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
} else {
docValue = tempBR.utf8ToString();
}
return compareValues(docValue, value);
return compareValues(topValue, docValue);
}
}

2
lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
View File

@ -632,7 +632,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
randomIOExceptionRateOnOpen = 0.0;
if (DirectoryReader.indexExists(this)) {
if (LuceneTestCase.VERBOSE) {
System.out.println("\nNOTE: MockDirectoryWrapper: now crash");
System.out.println("\nNOTE: MockDirectoryWrapper: now crush");
}
crash(); // corrupt any unsynced-files
if (LuceneTestCase.VERBOSE) {

11
solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
View File

@ -576,6 +576,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
return new FieldComparator<Integer>() {
private final int[] values = new int[numHits];
private int bottomVal;
private int topVal;
private TermsEnum termsEnum;
private DocsEnum docsEnum;
Set<String> seen = new HashSet<String>(elevations.ids.size());
@ -590,6 +591,11 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
bottomVal = values[slot];
}
@Override
public void setTopValue(Integer value) {
topVal = value.intValue();
}
private int docVal(int doc) {
if (ordSet.size() > 0) {
int slot = ordSet.find(doc);
@ -646,10 +652,9 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
}
@Override
public int compareDocToValue(int doc, Integer valueObj) {
final int value = valueObj.intValue();
public int compareTop(int doc) {
final int docValue = docVal(doc);
return docValue - value; // values will be small enough that there is no overflow concern
return topVal - docValue; // values will be small enough that there is no overflow concern
}
};
}

4
solr/core/src/java/org/apache/solr/schema/EnumField.java
View File

@ -178,7 +178,9 @@ public class EnumField extends PrimitiveFieldType {
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
final Object missingValue = Integer.MIN_VALUE;
return new SortField(field.getName(), FieldCache.NUMERIC_UTILS_INT_PARSER, top).setMissingValue(missingValue);
SortField sf = new SortField(field.getName(), FieldCache.NUMERIC_UTILS_INT_PARSER, top);
sf.setMissingValue(missingValue);
return sf;
}
/**

10
solr/core/src/java/org/apache/solr/schema/RandomSortField.java
View File

@ -109,6 +109,7 @@ public class RandomSortField extends FieldType {
int seed;
private final int[] values = new int[numHits];
int bottomVal;
int topVal;
@Override
public int compare(int slot1, int slot2) {
@ -120,6 +121,11 @@ public class RandomSortField extends FieldType {
bottomVal = values[slot];
}
@Override
public void setTopValue(Integer value) {
topVal = value.intValue();
}
@Override
public int compareBottom(int doc) {
return bottomVal - hash(doc+seed);
@ -142,9 +148,9 @@ public class RandomSortField extends FieldType {
}
@Override
public int compareDocToValue(int doc, Integer valueObj) {
public int compareTop(int doc) {
// values will be positive... no overflow possible.
return hash(doc+seed) - valueObj.intValue();
return topVal - hash(doc+seed);
}
};
}

20
solr/core/src/java/org/apache/solr/schema/TrieField.java
View File

@ -142,7 +142,9 @@ public class TrieField extends PrimitiveFieldType {
Object missingValue = null;
boolean sortMissingLast = field.sortMissingLast();
boolean sortMissingFirst = field.sortMissingFirst();
SortField sf;
switch (type) {
case INTEGER:
if( sortMissingLast ) {
@ -151,7 +153,9 @@ public class TrieField extends PrimitiveFieldType {
else if( sortMissingFirst ) {
missingValue = top ? Integer.MAX_VALUE : Integer.MIN_VALUE;
}
return new SortField( field.getName(), FieldCache.NUMERIC_UTILS_INT_PARSER, top).setMissingValue(missingValue);
sf = new SortField( field.getName(), FieldCache.NUMERIC_UTILS_INT_PARSER, top);
sf.setMissingValue(missingValue);
return sf;
case FLOAT:
if( sortMissingLast ) {
@ -160,7 +164,9 @@ public class TrieField extends PrimitiveFieldType {
else if( sortMissingFirst ) {
missingValue = top ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY;
}
return new SortField( field.getName(), FieldCache.NUMERIC_UTILS_FLOAT_PARSER, top).setMissingValue(missingValue);
sf = new SortField( field.getName(), FieldCache.NUMERIC_UTILS_FLOAT_PARSER, top);
sf.setMissingValue(missingValue);
return sf;
case DATE: // fallthrough
case LONG:
@ -170,7 +176,9 @@ public class TrieField extends PrimitiveFieldType {
else if( sortMissingFirst ) {
missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
}
return new SortField( field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, top).setMissingValue(missingValue);
sf = new SortField( field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, top);
sf.setMissingValue(missingValue);
return sf;
case DOUBLE:
if( sortMissingLast ) {
@ -179,7 +187,9 @@ public class TrieField extends PrimitiveFieldType {
else if( sortMissingFirst ) {
missingValue = top ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
}
return new SortField( field.getName(), FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, top).setMissingValue(missingValue);
sf = new SortField( field.getName(), FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, top);
sf.setMissingValue(missingValue);
return sf;
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + field.name);

69
solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java
View File

@ -87,6 +87,11 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
throw new UnsupportedOperationException();
}
@Override
public void setTopValue(Comparable value) {
throw new UnsupportedOperationException();
}
@Override
public int compareBottom(int doc) {
throw new UnsupportedOperationException();
@ -123,7 +128,7 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
}
@Override
public int compareDocToValue(int doc, Comparable docValue) {
public int compareTop(int doc) {
throw new UnsupportedOperationException();
}
@ -168,6 +173,11 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
return TermOrdValComparator_SML.createComparator(context.reader(), parent);
}
@Override
public void setTopValue(BytesRef value) {
throw new UnsupportedOperationException();
}
@Override
public int compare(int slot1, int slot2) {
if (readerGen[slot1] == readerGen[slot2]) {
@ -223,19 +233,16 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
return values==null ? parent.NULL_VAL : values[slot];
}
// this stuff caches the ordinal of the last 'value' from compareDocToValue
BytesRef lastValue = new BytesRef(); // sentinel
int lastOrd;
int lastReaderGen = -1;
boolean lastSameReader = false;
// nocommit, broken: nuke all of this
@Override
public int compareDocToValue(int doc, BytesRef value) {
int docOrd = termsIndex.getOrd(doc);
if (docOrd == -1) {
if (value == null) {
return 0;
}
return 1;
} else if (value == null) {
return -1;
}
termsIndex.lookupOrd(docOrd, tempBR);
return tempBR.compareTo(value);
public int compareTop(int doc) {
throw new UnsupportedOperationException();
}
}
@ -247,34 +254,18 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
int order = termsIndex.getOrd(doc);
if (order == -1) order = NULL_ORD;
int docOrd = termsIndex.getOrd(doc);
if (docOrd == -1) docOrd = NULL_ORD;
if (bottomSameReader) {
// ord is precisely comparable, even in the equal
// case
return bottomOrd - order;
// ord is precisely comparable, even in the equal case
return bottomOrd - docOrd;
} else if (bottomOrd >= docOrd) {
// the equals case always means bottom is > doc
// (because we set bottomOrd to the lower bound in
// setBottom):
return 1;
} else {
// ord is only approx comparable: if they are not
// equal, we can use that; if they are equal, we
// must fallback to compare by value
final int cmp = bottomOrd - order;
if (cmp != 0) {
return cmp;
}
// take care of the case where both vals are null
if (order == NULL_ORD) {
return 0;
}
// and at this point we know that neither value is null, so safe to compare
if (order == NULL_ORD) {
return bottomValue.compareTo(parent.NULL_VAL);
} else {
termsIndex.lookupOrd(order, tempBR);
return bottomValue.compareTo(tempBR);
}
return -1;
}
}