Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5399: add missingFirst/last support when sorting by Type.STRING; speed up deep paging; fix solr's distributed group sort for certain field types 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1558865 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2014-01-16 17:40:25 +00:00
parent f70138214b 6dc6525e8f
commit 7b34ac5c61
25 changed files with 853 additions and 649 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lucene/CHANGES.txt
View File

@ -97,6 +97,12 @@ New Features
AnalyzingInfixSuggester but boosts suggestions that matched tokens
with lower positions. (Remi Melisson via Mike McCandless)
* LUCENE-4399: When sorting by String (SortField.STRING), you can now
specify whether missing values should be sorted first (the default),
using SortField.setMissingValue(SortField.STRING_FIRST), or last,
using SortField.setMissingValue(SortField.STRING_LAST). (Rob Muir,
Mike McCandless)
Build
@ -183,6 +189,9 @@ Changes in Runtime Behavior
AlreadyClosedException if the refCount in incremented but
is less that 1. (Simon Willnauer)
* LUCENE-4399: Deep paging using IndexSearcher.searchAfter when
sorting by fields is faster (Rob Muir, Mike McCandless)
Documentation
* LUCENE-5384: Add some tips for making tokenfilters and tokenizers

424
lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
View File

@ -62,6 +62,18 @@ import org.apache.lucene.util.BytesRef;
* <li> {@link #compareBottom} Compare a new hit (docID)
* against the "weakest" (bottom) entry in the queue.
*
* <li> {@link #setTopValue} This method is called by
* {@link TopFieldCollector} to notify the
* FieldComparator of the top most value, which is
* used by future calls to {@link #compareTop}.
*
* <li> {@link #compareBottom} Compare a new hit (docID)
* against the "weakest" (bottom) entry in the queue.
*
* <li> {@link #compareTop} Compare a new hit (docID)
* against the top value previously set by a call to
* {@link #setTopValue}.
*
* <li> {@link #copy} Installs a new hit into the
* priority queue. The {@link FieldValueHitQueue}
* calls this method when a new hit is competitive.
@ -104,7 +116,15 @@ public abstract class FieldComparator<T> {
public abstract void setBottom(final int slot);
/**
* Compare the bottom of the queue with doc. This will
* Record the top value, for future calls to {@link
* #compareTop}. This is only called for searches that
* use searchAfter (deep paging), and is called before any
* calls to {@link #setNextReader}.
*/
public abstract void setTopValue(T value);
/**
* Compare the bottom of the queue with this doc. This will
* only invoked after setBottom has been called. This
* should return the same result as {@link
* #compare(int,int)}} as if bottom were slot1 and the new
@ -122,6 +142,22 @@ public abstract class FieldComparator<T> {
*/
public abstract int compareBottom(int doc) throws IOException;
/**
* Compare the top value with this doc. This will
* only invoked after setTopValue has been called. This
* should return the same result as {@link
* #compare(int,int)}} as if topValue were slot1 and the new
* document were slot 2. This is only called for searches that
* use searchAfter (deep paging).
*
* @param doc that was hit
* @return any N < 0 if the doc's value is sorted after
* the bottom entry (not competitive), any N > 0 if the
* doc's value is sorted before the bottom entry and 0 if
* they are equal.
*/
public abstract int compareTop(int doc) throws IOException;
/**
* This method is called when a new hit is competitive.
* You should copy any state associated with this document
@ -184,10 +220,6 @@ public abstract class FieldComparator<T> {
}
}
/** Returns negative result if the doc's value is less
* than the provided value. */
public abstract int compareDocToValue(int doc, T value) throws IOException;
/**
* Base FieldComparator class for numeric types
*/
@ -223,6 +255,7 @@ public abstract class FieldComparator<T> {
private final DoubleParser parser;
private FieldCache.Doubles currentReaderValues;
private double bottom;
private double topValue;
DoubleComparator(int numHits, String field, FieldCache.Parser parser, Double missingValue) {
super(field, missingValue);
@ -272,21 +305,25 @@ public abstract class FieldComparator<T> {
this.bottom = values[bottom];
}
@Override
public void setTopValue(Double value) {
topValue = value;
}
@Override
public Double value(int slot) {
return Double.valueOf(values[slot]);
}
@Override
public int compareDocToValue(int doc, Double valueObj) {
final double value = valueObj.doubleValue();
public int compareTop(int doc) {
double docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
docValue = missingValue;
}
return Double.compare(docValue, value);
return Double.compare(topValue, docValue);
}
}
@ -297,6 +334,7 @@ public abstract class FieldComparator<T> {
private final FloatParser parser;
private FieldCache.Floats currentReaderValues;
private float bottom;
private float topValue;
FloatComparator(int numHits, String field, FieldCache.Parser parser, Float missingValue) {
super(field, missingValue);
@ -347,21 +385,25 @@ public abstract class FieldComparator<T> {
this.bottom = values[bottom];
}
@Override
public void setTopValue(Float value) {
topValue = value;
}
@Override
public Float value(int slot) {
return Float.valueOf(values[slot]);
}
@Override
public int compareDocToValue(int doc, Float valueObj) {
final float value = valueObj.floatValue();
public int compareTop(int doc) {
float docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
docValue = missingValue;
}
return Float.compare(docValue, value);
return Float.compare(topValue, docValue);
}
}
@ -372,6 +414,7 @@ public abstract class FieldComparator<T> {
private final IntParser parser;
private FieldCache.Ints currentReaderValues;
private int bottom; // Value of bottom of queue
private int topValue;
IntComparator(int numHits, String field, FieldCache.Parser parser, Integer missingValue) {
super(field, missingValue);
@ -421,21 +464,25 @@ public abstract class FieldComparator<T> {
this.bottom = values[bottom];
}
@Override
public void setTopValue(Integer value) {
topValue = value;
}
@Override
public Integer value(int slot) {
return Integer.valueOf(values[slot]);
}
@Override
public int compareDocToValue(int doc, Integer valueObj) {
final int value = valueObj.intValue();
public int compareTop(int doc) {
int docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
docValue = missingValue;
}
return Integer.compare(docValue, value);
return Integer.compare(topValue, docValue);
}
}
@ -446,6 +493,7 @@ public abstract class FieldComparator<T> {
private final LongParser parser;
private FieldCache.Longs currentReaderValues;
private long bottom;
private long topValue;
LongComparator(int numHits, String field, FieldCache.Parser parser, Long missingValue) {
super(field, missingValue);
@ -497,21 +545,25 @@ public abstract class FieldComparator<T> {
this.bottom = values[bottom];
}
@Override
public void setTopValue(Long value) {
topValue = value;
}
@Override
public Long value(int slot) {
return Long.valueOf(values[slot]);
}
@Override
public int compareDocToValue(int doc, Long valueObj) {
final long value = valueObj.longValue();
public int compareTop(int doc) {
long docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
docValue = missingValue;
}
return Long.compare(docValue, value);
return Long.compare(topValue, docValue);
}
}
@ -525,7 +577,8 @@ public abstract class FieldComparator<T> {
private final float[] scores;
private float bottom;
private Scorer scorer;
private float topValue;
RelevanceComparator(int numHits) {
scores = new float[numHits];
}
@ -558,6 +611,11 @@ public abstract class FieldComparator<T> {
this.bottom = scores[bottom];
}
@Override
public void setTopValue(Float value) {
topValue = value;
}
@Override
public void setScorer(Scorer scorer) {
// wrap with a ScoreCachingWrappingScorer so that successive calls to
@ -584,11 +642,10 @@ public abstract class FieldComparator<T> {
}
@Override
public int compareDocToValue(int doc, Float valueObj) throws IOException {
final float value = valueObj.floatValue();
public int compareTop(int doc) throws IOException {
float docValue = scorer.score();
assert !Float.isNaN(docValue);
return Float.compare(value, docValue);
return Float.compare(docValue, topValue);
}
}
@ -597,6 +654,7 @@ public abstract class FieldComparator<T> {
private final int[] docIDs;
private int docBase;
private int bottom;
private int topValue;
DocComparator(int numHits) {
docIDs = new int[numHits];
@ -633,16 +691,20 @@ public abstract class FieldComparator<T> {
this.bottom = docIDs[bottom];
}
@Override
public void setTopValue(Integer value) {
topValue = value;
}
@Override
public Integer value(int slot) {
return Integer.valueOf(docIDs[slot]);
}
@Override
public int compareDocToValue(int doc, Integer valueObj) {
final int value = valueObj.intValue();
public int compareTop(int doc) {
int docValue = docBase + doc;
return Integer.compare(docValue, value);
return Integer.compare(topValue, docValue);
}
}
@ -700,13 +762,42 @@ public abstract class FieldComparator<T> {
@lucene.internal */
BytesRef bottomValue;
/** Set by setTopValue. */
BytesRef topValue;
boolean topSameReader;
int topOrd;
private int docBase;
final BytesRef tempBR = new BytesRef();
/** -1 if missing values are sorted first, 1 if they are
* sorted last */
final int missingSortCmp;
/** Which ordinal to use for a missing value. */
final int missingOrd;
/** Creates this, sorting missing values first. */
public TermOrdValComparator(int numHits, String field) {
this(numHits, field, false);
}
/** Creates this, with control over how missing values
* are sorted. Pass sortMissingLast=true to put
* missing values at the end. */
public TermOrdValComparator(int numHits, String field, boolean sortMissingLast) {
ords = new int[numHits];
values = new BytesRef[numHits];
readerGen = new int[numHits];
this.field = field;
if (sortMissingLast) {
missingSortCmp = 1;
missingOrd = Integer.MAX_VALUE;
} else {
missingSortCmp = -1;
missingOrd = -1;
}
}
@Override
@ -721,140 +812,78 @@ public abstract class FieldComparator<T> {
if (val2 == null) {
return 0;
}
return -1;
return missingSortCmp;
} else if (val2 == null) {
return 1;
return -missingSortCmp;
}
return val1.compareTo(val2);
}
@Override
public int compareBottom(int doc) {
throw new UnsupportedOperationException();
assert bottomSlot != -1;
int docOrd = termsIndex.getOrd(doc);
if (docOrd == -1) {
docOrd = missingOrd;
}
if (bottomSameReader) {
// ord is precisely comparable, even in the equal case
return bottomOrd - docOrd;
} else if (bottomOrd >= docOrd) {
// the equals case always means bottom is > doc
// (because we set bottomOrd to the lower bound in
// setBottom):
return 1;
} else {
return -1;
}
}
@Override
public void copy(int slot, int doc) {
throw new UnsupportedOperationException();
}
@Override
public int compareDocToValue(int doc, BytesRef value) {
int ord = termsIndex.getOrd(doc);
if (ord == -1) {
if (value == null) {
return 0;
ord = missingOrd;
values[slot] = null;
} else {
assert ord >= 0;
if (values[slot] == null) {
values[slot] = new BytesRef();
}
return -1;
} else if (value == null) {
return 1;
termsIndex.lookupOrd(ord, values[slot]);
}
termsIndex.lookupOrd(ord, tempBR);
return tempBR.compareTo(value);
ords[slot] = ord;
readerGen[slot] = currentReaderGen;
}
/** Base class for specialized (per bit width of the
* ords) per-segment comparator. NOTE: this is messy;
* we do this only because hotspot can't reliably inline
* the underlying array access when looking up doc->ord
* @lucene.internal
*/
abstract class PerSegmentComparator extends FieldComparator<BytesRef> {
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
return TermOrdValComparator.this.setNextReader(context);
}
@Override
public int compare(int slot1, int slot2) {
return TermOrdValComparator.this.compare(slot1, slot2);
}
@Override
public void setBottom(final int bottom) {
TermOrdValComparator.this.setBottom(bottom);
}
@Override
public BytesRef value(int slot) {
return TermOrdValComparator.this.value(slot);
}
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
if (val1 == null) {
if (val2 == null) {
return 0;
}
return -1;
} else if (val2 == null) {
return 1;
}
return val1.compareTo(val2);
}
@Override
public int compareDocToValue(int doc, BytesRef value) {
return TermOrdValComparator.this.compareDocToValue(doc, value);
}
}
// Used per-segment when docToOrd is null:
private final class AnyOrdComparator extends PerSegmentComparator {
private final SortedDocValues termsIndex;
private final int docBase;
public AnyOrdComparator(SortedDocValues termsIndex, int docBase) {
this.termsIndex = termsIndex;
this.docBase = docBase;
}
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
final int docOrd = termsIndex.getOrd(doc);
if (bottomSameReader) {
// ord is precisely comparable, even in the equal case
return bottomOrd - docOrd;
} else if (bottomOrd >= docOrd) {
// the equals case always means bottom is > doc
// (because we set bottomOrd to the lower bound in
// setBottom):
return 1;
} else {
return -1;
}
}
@Override
public void copy(int slot, int doc) {
final int ord = termsIndex.getOrd(doc);
ords[slot] = ord;
if (ord == -1) {
values[slot] = null;
} else {
assert ord >= 0;
if (values[slot] == null) {
values[slot] = new BytesRef();
}
termsIndex.lookupOrd(ord, values[slot]);
}
readerGen[slot] = currentReaderGen;
}
}
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
final int docBase = context.docBase;
docBase = context.docBase;
termsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), field);
FieldComparator<BytesRef> perSegComp = new AnyOrdComparator(termsIndex, docBase);
currentReaderGen++;
if (topValue != null) {
// Recompute topOrd/SameReader
int ord = termsIndex.lookupTerm(topValue);
if (ord >= 0) {
topSameReader = true;
topOrd = ord;
} else {
topSameReader = false;
topOrd = -ord-2;
}
} else {
topOrd = missingOrd;
topSameReader = true;
}
//System.out.println(" setNextReader topOrd=" + topOrd + " topSameReader=" + topSameReader);
if (bottomSlot != -1) {
perSegComp.setBottom(bottomSlot);
// Recompute bottomOrd/SameReader
setBottom(bottomSlot);
}
return perSegComp;
return this;
}
@Override
@ -867,18 +896,18 @@ public abstract class FieldComparator<T> {
bottomSameReader = true;
} else {
if (bottomValue == null) {
// -1 ord is null for all segments
assert ords[bottomSlot] == -1;
bottomOrd = -1;
// missingOrd is null for all segments
assert ords[bottomSlot] == missingOrd;
bottomOrd = missingOrd;
bottomSameReader = true;
readerGen[bottomSlot] = currentReaderGen;
} else {
final int index = termsIndex.lookupTerm(bottomValue);
if (index < 0) {
bottomOrd = -index - 2;
final int ord = termsIndex.lookupTerm(bottomValue);
if (ord < 0) {
bottomOrd = -ord - 2;
bottomSameReader = false;
} else {
bottomOrd = index;
bottomOrd = ord;
// exact value match
bottomSameReader = true;
readerGen[bottomSlot] = currentReaderGen;
@ -888,28 +917,77 @@ public abstract class FieldComparator<T> {
}
}
@Override
public void setTopValue(BytesRef value) {
// null is fine: it means the last doc of the prior
// search was missing this value
topValue = value;
//System.out.println("setTopValue " + topValue);
}
@Override
public BytesRef value(int slot) {
return values[slot];
}
@Override
public int compareTop(int doc) {
int ord = termsIndex.getOrd(doc);
if (ord == -1) {
ord = missingOrd;
}
if (topSameReader) {
// ord is precisely comparable, even in the equal
// case
//System.out.println("compareTop doc=" + doc + " ord=" + ord + " ret=" + (topOrd-ord));
return topOrd - ord;
} else if (ord <= topOrd) {
// the equals case always means doc is < value
// (because we set lastOrd to the lower bound)
return 1;
} else {
return -1;
}
}
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
if (val1 == null) {
if (val2 == null) {
return 0;
}
return missingSortCmp;
} else if (val2 == null) {
return -missingSortCmp;
}
return val1.compareTo(val2);
}
}
// just used internally in this comparator
private static final byte[] MISSING_BYTES = new byte[0];
/** Sorts by field's natural Term sort order. All
* comparisons are done using BytesRef.compareTo, which is
* slow for medium to large result sets but possibly
* very fast for very small results sets. */
// TODO: should we remove this? who really uses it?
public static final class TermValComparator extends FieldComparator<BytesRef> {
// sentinels, just used internally in this comparator
private static final byte[] MISSING_BYTES = new byte[0];
private static final byte[] NON_MISSING_BYTES = new byte[0];
private BytesRef[] values;
private BinaryDocValues docTerms;
private Bits docsWithField;
private final String field;
private BytesRef bottom;
private BytesRef topValue;
private final BytesRef tempBR = new BytesRef();
// TODO: add missing first/last support here?
/** Sole constructor. */
TermValComparator(int numHits, String field) {
values = new BytesRef[numHits];
this.field = field;
@ -919,12 +997,12 @@ public abstract class FieldComparator<T> {
public int compare(int slot1, int slot2) {
final BytesRef val1 = values[slot1];
final BytesRef val2 = values[slot2];
if (val1 == null) {
if (val2 == null) {
if (val1.bytes == MISSING_BYTES) {
if (val2.bytes == MISSING_BYTES) {
return 0;
}
return -1;
} else if (val2 == null) {
} else if (val2.bytes == MISSING_BYTES) {
return 1;
}
@ -934,18 +1012,8 @@ public abstract class FieldComparator<T> {
@Override
public int compareBottom(int doc) {
docTerms.get(doc, tempBR);
if (tempBR.length == 0 && docsWithField.get(doc) == false) {
tempBR.bytes = MISSING_BYTES;
}
if (bottom.bytes == MISSING_BYTES) {
if (tempBR.bytes == MISSING_BYTES) {
return 0;
}
return -1;
} else if (tempBR.bytes == MISSING_BYTES) {
return 1;
}
return bottom.compareTo(tempBR);
setMissingBytes(doc, tempBR);
return compareValues(bottom, tempBR);
}
@Override
@ -954,9 +1022,7 @@ public abstract class FieldComparator<T> {
values[slot] = new BytesRef();
}
docTerms.get(doc, values[slot]);
if (values[slot].length == 0 && docsWithField.get(doc) == false) {
values[slot].bytes = MISSING_BYTES;
}
setMissingBytes(doc, values[slot]);
}
@Override
@ -971,6 +1037,14 @@ public abstract class FieldComparator<T> {
this.bottom = values[bottom];
}
@Override
public void setTopValue(BytesRef value) {
if (value == null) {
throw new IllegalArgumentException("value cannot be null");
}
topValue = value;
}
@Override
public BytesRef value(int slot) {
return values[slot];
@ -978,24 +1052,34 @@ public abstract class FieldComparator<T> {
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
if (val1 == null) {
if (val2 == null) {
// missing always sorts first:
if (val1.bytes == MISSING_BYTES) {
if (val2.bytes == MISSING_BYTES) {
return 0;
}
return -1;
} else if (val2 == null) {
} else if (val2.bytes == MISSING_BYTES) {
return 1;
}
return val1.compareTo(val2);
}
@Override
public int compareDocToValue(int doc, BytesRef value) {
public int compareTop(int doc) {
docTerms.get(doc, tempBR);
if (tempBR.length == 0 && docsWithField.get(doc) == false) {
tempBR.bytes = MISSING_BYTES;
setMissingBytes(doc, tempBR);
return compareValues(topValue, tempBR);
}
private void setMissingBytes(int doc, BytesRef br) {
if (br.length == 0) {
br.offset = 0;
if (docsWithField.get(doc) == false) {
br.bytes = MISSING_BYTES;
} else {
br.bytes = NON_MISSING_BYTES;
}
}
return tempBR.compareTo(value);
}
}
}

12
lucene/core/src/java/org/apache/lucene/search/FieldDoc.java
View File

@ -17,6 +17,8 @@ package org.apache.lucene.search;
* limitations under the License.
*/
import java.util.Arrays;
/**
* Expert: A ScoreDoc which also contains information about
* how to sort the referenced document. In addition to the
@ -69,14 +71,10 @@ public class FieldDoc extends ScoreDoc {
@Override
public String toString() {
// super.toString returns the doc and score information, so just add the
// fields information
// fields information
StringBuilder sb = new StringBuilder(super.toString());
sb.append("[");
for (int i = 0; i < fields.length; i++) {
sb.append(fields[i]).append(", ");
}
sb.setLength(sb.length() - 2); // discard last ", "
sb.append("]");
sb.append(" fields=");
sb.append(Arrays.toString(fields));
return sb.toString();
}
}

39
lucene/core/src/java/org/apache/lucene/search/SortField.java
View File

@ -106,6 +106,9 @@ public class SortField {
// Used for 'sortMissingFirst/Last'
public Object missingValue = null;
// Only used with type=STRING
public boolean sortMissingLast;
/** Creates a sort by terms in the given field with the type of term
* values explicitly given.
* @param field Name of field to sort by. Can be <code>null</code> if
@ -165,13 +168,34 @@ public class SortField {
this.reverse = reverse;
this.parser = parser;
}
/** Pass this to {@link #setMissingValue} to have missing
* string values sort first. */
public final static Object STRING_FIRST = new Object() {
@Override
public String toString() {
return "SortField.STRING_FIRST";
}
};
public SortField setMissingValue(Object missingValue) {
if (type != Type.INT && type != Type.FLOAT && type != Type.LONG && type != Type.DOUBLE) {
throw new IllegalArgumentException( "Missing value only works for numeric types" );
/** Pass this to {@link #setMissingValue} to have missing
* string values sort last. */
public final static Object STRING_LAST = new Object() {
@Override
public String toString() {
return "SortField.STRING_LAST";
}
};
public void setMissingValue(Object missingValue) {
if (type == Type.STRING) {
if (missingValue != STRING_FIRST && missingValue != STRING_LAST) {
throw new IllegalArgumentException("For STRING type, missing value must be either STRING_FIRST or STRING_LAST");
}
} else if (type != Type.INT && type != Type.FLOAT && type != Type.LONG && type != Type.DOUBLE) {
throw new IllegalArgumentException("Missing value only works for numeric or STRING types");
}
this.missingValue = missingValue;
return this;
}
/** Creates a sort with a custom comparison function.
@ -294,6 +318,10 @@ public class SortField {
}
if (reverse) buffer.append('!');
if (missingValue != null) {
buffer.append(" missingValue=");
buffer.append(missingValue);
}
return buffer.toString();
}
@ -376,9 +404,10 @@ public class SortField {
return comparatorSource.newComparator(field, numHits, sortPos, reverse);
case STRING:
return new FieldComparator.TermOrdValComparator(numHits, field);
return new FieldComparator.TermOrdValComparator(numHits, field, missingValue == STRING_LAST);
case STRING_VAL:
// TODO: should we remove this? who really uses it?
return new FieldComparator.TermValComparator(numHits, field);
case REWRITEABLE:

70
lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java
View File

@ -869,6 +869,13 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
// Must set maxScore to NEG_INF, or otherwise Math.max always returns NaN.
maxScore = Float.NEGATIVE_INFINITY;
// Tell all comparators their top value:
for(int i=0;i<comparators.length;i++) {
@SuppressWarnings("unchecked")
FieldComparator<Object> comparator = (FieldComparator<Object>) comparators[i];
comparator.setTopValue(after.fields[i]);
}
}
void updateBottom(int doc, float score) {
@ -880,37 +887,9 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
@SuppressWarnings({"unchecked", "rawtypes"})
@Override
public void collect(int doc) throws IOException {
totalHits++;
//System.out.println(" collect doc=" + doc);
// Check if this hit was already collected on a
// previous page:
boolean sameValues = true;
for(int compIDX=0;compIDX<comparators.length;compIDX++) {
final FieldComparator comp = comparators[compIDX];
final int cmp = reverseMul[compIDX] * comp.compareDocToValue(doc, after.fields[compIDX]);
if (cmp < 0) {
// Already collected on a previous page
//System.out.println(" skip: before");
return;
} else if (cmp > 0) {
// Not yet collected
sameValues = false;
//System.out.println(" keep: after");
break;
}
}
// Tie-break by docID:
if (sameValues && doc <= afterDoc) {
// Already collected on a previous page
//System.out.println(" skip: tie-break");
return;
}
collectedHits++;
totalHits++;
float score = Float.NaN;
if (trackMaxScore) {
@ -921,7 +900,8 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
}
if (queueFull) {
// Fastmatch: return if this hit is not competitive
// Fastmatch: return if this hit is no better than
// the worst hit currently in the queue:
for (int i = 0;; i++) {
final int c = reverseMul[i] * comparators[i].compareBottom(doc);
if (c < 0) {
@ -939,7 +919,35 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
break;
}
}
}
// Check if this hit was already collected on a
// previous page:
boolean sameValues = true;
for(int compIDX=0;compIDX<comparators.length;compIDX++) {
final FieldComparator comp = comparators[compIDX];
final int cmp = reverseMul[compIDX] * comp.compareTop(doc);
if (cmp > 0) {
// Already collected on a previous page
//System.out.println(" skip: before");
return;
} else if (cmp < 0) {
// Not yet collected
sameValues = false;
//System.out.println(" keep: after; reverseMul=" + reverseMul[compIDX]);
break;
}
}
// Tie-break by docID:
if (sameValues && doc <= afterDoc) {
// Already collected on a previous page
//System.out.println(" skip: tie-break");
return;
}
if (queueFull) {
// This hit is competitive - replace bottom element in queue & adjustTop
for (int i = 0; i < comparators.length; i++) {
comparators[i].copy(bottom.slot, doc);
@ -955,6 +963,8 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
comparators[i].setBottom(bottom.slot);
}
} else {
collectedHits++;
// Startup transient: queue hasn't gathered numHits yet
final int slot = collectedHits - 1;
//System.out.println(" slot=" + slot);

2
lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java
View File

@ -96,7 +96,7 @@ package org.apache.lucene.util;
public final class UnicodeUtil {
/** A binary term consisting of a number of 0xff bytes, likely to be bigger than other terms
* one would normally encounter, and definitely bigger than any UTF-8 terms.
* (e.g. collation keys) one would normally encounter, and definitely bigger than any UTF-8 terms.
* <p>
* WARNING: This is not a valid UTF8 Term
**/

7
lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
View File

@ -145,6 +145,11 @@ final class JustCompileSearch {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
@Override
public void setTopValue(Object value) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
@Override
public FieldComparator<Object> setNextReader(AtomicReaderContext context) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
@ -156,7 +161,7 @@ final class JustCompileSearch {
}
@Override
public int compareDocToValue(int doc, Object value) {
public int compareTop(int doc) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
}

12
lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java
View File

@ -157,6 +157,11 @@ class ElevationComparatorSource extends FieldComparatorSource {
bottomVal = values[slot];
}
@Override
public void setTopValue(Integer value) {
throw new UnsupportedOperationException();
}
private int docVal(int doc) {
int ord = idIndex.getOrd(doc);
if (ord == -1) {
@ -190,11 +195,8 @@ class ElevationComparatorSource extends FieldComparatorSource {
}
@Override
public int compareDocToValue(int doc, Integer valueObj) {
final int value = valueObj.intValue();
final int docValue = docVal(doc);
// values will be small enough that there is no overflow concern
return value - docValue;
public int compareTop(int doc) {
throw new UnsupportedOperationException();
}
};
}

179
lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java
View File

@ -17,7 +17,10 @@ package org.apache.lucene.search;
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
@ -29,6 +32,7 @@ import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@ -45,39 +49,127 @@ public class TestSearchAfter extends LuceneTestCase {
private Directory dir;
private IndexReader reader;
private IndexSearcher searcher;
private int iter;
private List<SortField> allSortFields;
@Override
public void setUp() throws Exception {
super.setUp();
allSortFields = new ArrayList<SortField>(Arrays.asList(new SortField[] {
new SortField("int", SortField.Type.INT, false),
new SortField("long", SortField.Type.LONG, false),
new SortField("float", SortField.Type.FLOAT, false),
new SortField("double", SortField.Type.DOUBLE, false),
new SortField("bytes", SortField.Type.STRING, false),
new SortField("bytesval", SortField.Type.STRING_VAL, false),
new SortField("intdocvalues", SortField.Type.INT, false),
new SortField("floatdocvalues", SortField.Type.FLOAT, false),
new SortField("sortedbytesdocvalues", SortField.Type.STRING, false),
new SortField("sortedbytesdocvaluesval", SortField.Type.STRING_VAL, false),
new SortField("straightbytesdocvalues", SortField.Type.STRING_VAL, false),
new SortField("int", SortField.Type.INT, true),
new SortField("long", SortField.Type.LONG, true),
new SortField("float", SortField.Type.FLOAT, true),
new SortField("double", SortField.Type.DOUBLE, true),
new SortField("bytes", SortField.Type.STRING, true),
new SortField("bytesval", SortField.Type.STRING_VAL, true),
new SortField("intdocvalues", SortField.Type.INT, true),
new SortField("floatdocvalues", SortField.Type.FLOAT, true),
new SortField("sortedbytesdocvalues", SortField.Type.STRING, true),
new SortField("sortedbytesdocvaluesval", SortField.Type.STRING_VAL, true),
new SortField("straightbytesdocvalues", SortField.Type.STRING_VAL, true),
SortField.FIELD_SCORE,
SortField.FIELD_DOC,
}));
// Also test missing first / last for the "string" sorts:
for(String field : new String[] {"bytes", "sortedbytesdocvalues"}) {
for(int rev=0;rev<2;rev++) {
boolean reversed = rev == 0;
SortField sf = new SortField(field, SortField.Type.STRING, reversed);
sf.setMissingValue(SortField.STRING_FIRST);
allSortFields.add(sf);
sf = new SortField(field, SortField.Type.STRING, reversed);
sf.setMissingValue(SortField.STRING_LAST);
allSortFields.add(sf);
}
}
int limit = allSortFields.size();
for(int i=0;i<limit;i++) {
SortField sf = allSortFields.get(i);
if (sf.getType() == SortField.Type.INT) {
SortField sf2 = new SortField(sf.getField(), SortField.Type.INT, sf.getReverse());
sf2.setMissingValue(random().nextInt());
allSortFields.add(sf2);
} else if (sf.getType() == SortField.Type.LONG) {
SortField sf2 = new SortField(sf.getField(), SortField.Type.LONG, sf.getReverse());
sf2.setMissingValue(random().nextLong());
allSortFields.add(sf2);
} else if (sf.getType() == SortField.Type.FLOAT) {
SortField sf2 = new SortField(sf.getField(), SortField.Type.FLOAT, sf.getReverse());
sf2.setMissingValue(random().nextFloat());
allSortFields.add(sf2);
} else if (sf.getType() == SortField.Type.DOUBLE) {
SortField sf2 = new SortField(sf.getField(), SortField.Type.DOUBLE, sf.getReverse());
sf2.setMissingValue(random().nextDouble());
allSortFields.add(sf2);
}
}
dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
int numDocs = atLeast(200);
for (int i = 0; i < numDocs; i++) {
List<Field> fields = new ArrayList<Field>();
fields.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
fields.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
fields.add(newStringField("byte", "" + ((byte) random().nextInt()), Field.Store.NO));
fields.add(newStringField("short", "" + ((short) random().nextInt()), Field.Store.NO));
fields.add(new IntField("int", random().nextInt(), Field.Store.NO));
fields.add(new LongField("long", random().nextLong(), Field.Store.NO));
fields.add(new FloatField("float", random().nextFloat(), Field.Store.NO));
fields.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
fields.add(newStringField("bytes", _TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
fields.add(newStringField("bytesval", _TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
fields.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
fields.add(new NumericDocValuesField("intdocvalues", random().nextInt()));
fields.add(new FloatDocValuesField("floatdocvalues", random().nextFloat()));
fields.add(new SortedDocValuesField("sortedbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
fields.add(new SortedDocValuesField("sortedbytesdocvaluesval", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
fields.add(new BinaryDocValuesField("straightbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
Document document = new Document();
document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
document.add(newStringField("byte", "" + ((byte) random().nextInt()), Field.Store.NO));
document.add(newStringField("short", "" + ((short) random().nextInt()), Field.Store.NO));
document.add(new IntField("int", random().nextInt(), Field.Store.NO));
document.add(new LongField("long", random().nextLong(), Field.Store.NO));
document.add(new FloatField("float", random().nextFloat(), Field.Store.NO));
document.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
document.add(newStringField("bytes", _TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
document.add(newStringField("bytesval", _TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO));
document.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
document.add(new NumericDocValuesField("intdocvalues", random().nextInt()));
document.add(new FloatDocValuesField("floatdocvalues", random().nextFloat()));
document.add(new SortedDocValuesField("sortedbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
document.add(new SortedDocValuesField("sortedbytesdocvaluesval", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
document.add(new BinaryDocValuesField("straightbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random()))));
document.add(new StoredField("id", ""+i));
if (VERBOSE) {
System.out.println(" add doc id=" + i);
}
for(Field field : fields) {
// So we are sometimes missing that field:
if (random().nextInt(5) != 4) {
document.add(field);
if (VERBOSE) {
System.out.println(" " + field);
}
}
}
iw.addDocument(document);
if (random().nextInt(50) == 17) {
iw.commit();
}
}
reader = iw.getReader();
iw.close();
searcher = newSearcher(reader);
if (VERBOSE) {
System.out.println(" searcher=" + searcher);
}
}
@Override
@ -104,25 +196,25 @@ public class TestSearchAfter extends LuceneTestCase {
assertQuery(bq, null);
}
}
void assertQuery(Query query, Filter filter) throws Exception {
assertQuery(query, filter, null);
assertQuery(query, filter, Sort.RELEVANCE);
assertQuery(query, filter, Sort.INDEXORDER);
for(int rev=0;rev<2;rev++) {
boolean reversed = rev == 1;
assertQuery(query, filter, new Sort(new SortField[] {new SortField("int", SortField.Type.INT, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("long", SortField.Type.LONG, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("float", SortField.Type.FLOAT, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("double", SortField.Type.DOUBLE, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("bytes", SortField.Type.STRING, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("bytesval", SortField.Type.STRING_VAL, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("intdocvalues", SortField.Type.INT, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("floatdocvalues", SortField.Type.FLOAT, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("sortedbytesdocvalues", SortField.Type.STRING, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("sortedbytesdocvaluesval", SortField.Type.STRING_VAL, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("straightbytesdocvalues", SortField.Type.STRING_VAL, reversed)}));
for(SortField sortField : allSortFields) {
assertQuery(query, filter, new Sort(new SortField[] {sortField}));
}
for(int i=0;i<20;i++) {
assertQuery(query, filter, getRandomSort());
}
}
Sort getRandomSort() {
SortField[] sortFields = new SortField[_TestUtil.nextInt(random(), 2, 7)];
for(int i=0;i<sortFields.length;i++) {
sortFields[i] = allSortFields.get(random().nextInt(allSortFields.size()));
}
return new Sort(sortFields);
}
void assertQuery(Query query, Filter filter, Sort sort) throws Exception {
@ -130,18 +222,23 @@ public class TestSearchAfter extends LuceneTestCase {
TopDocs all;
int pageSize = _TestUtil.nextInt(random(), 1, maxDoc*2);
if (VERBOSE) {
System.out.println("\nassertQuery: query=" + query + " filter=" + filter + " sort=" + sort + " pageSize=" + pageSize);
System.out.println("\nassertQuery " + (iter++) + ": query=" + query + " filter=" + filter + " sort=" + sort + " pageSize=" + pageSize);
}
final boolean doMaxScore = random().nextBoolean();
final boolean doScores = random().nextBoolean();
if (sort == null) {
all = searcher.search(query, filter, maxDoc);
} else if (sort == Sort.RELEVANCE) {
all = searcher.search(query, filter, maxDoc, sort, true, doMaxScore);
} else {
all = searcher.search(query, filter, maxDoc, sort);
all = searcher.search(query, filter, maxDoc, sort, doScores, doMaxScore);
}
if (VERBOSE) {
System.out.println(" all.totalHits=" + all.totalHits);
int upto = 0;
for(ScoreDoc scoreDoc : all.scoreDocs) {
System.out.println(" hit " + (upto++) + ": id=" + searcher.doc(scoreDoc.doc).get("id") + " " + scoreDoc);
}
}
int pageStart = 0;
ScoreDoc lastBottom = null;
@ -154,14 +251,17 @@ public class TestSearchAfter extends LuceneTestCase {
paged = searcher.searchAfter(lastBottom, query, filter, pageSize);
} else {
if (VERBOSE) {
System.out.println(" iter lastBottom=" + lastBottom + (lastBottom == null ? "" : " fields=" + Arrays.toString(((FieldDoc) lastBottom).fields)));
System.out.println(" iter lastBottom=" + lastBottom);
}
if (sort == Sort.RELEVANCE) {
paged = searcher.searchAfter(lastBottom, query, filter, pageSize, sort, true, doMaxScore);
} else {
paged = searcher.searchAfter(lastBottom, query, filter, pageSize, sort);
paged = searcher.searchAfter(lastBottom, query, filter, pageSize, sort, doScores, doMaxScore);
}
}
if (VERBOSE) {
System.out.println(" " + paged.scoreDocs.length + " hits on page");
}
if (paged.scoreDocs.length == 0) {
break;
@ -173,11 +273,16 @@ public class TestSearchAfter extends LuceneTestCase {
assertEquals(all.scoreDocs.length, pageStart);
}
static void assertPage(int pageStart, TopDocs all, TopDocs paged) {
void assertPage(int pageStart, TopDocs all, TopDocs paged) throws IOException {
assertEquals(all.totalHits, paged.totalHits);
for (int i = 0; i < paged.scoreDocs.length; i++) {
ScoreDoc sd1 = all.scoreDocs[pageStart + i];
ScoreDoc sd2 = paged.scoreDocs[i];
if (VERBOSE) {
System.out.println(" hit " + (pageStart + i));
System.out.println(" expected id=" + searcher.doc(sd1.doc).get("id") + " " + sd1);
System.out.println(" actual id=" + searcher.doc(sd2.doc).get("id") + " " + sd2);
}
assertEquals(sd1.doc, sd2.doc);
assertEquals(sd1.score, sd2.score, 0f);
if (sd1 instanceof FieldDoc) {

126
lucene/core/src/test/org/apache/lucene/search/TestSort.java
View File

@ -196,6 +196,132 @@ public class TestSort extends LuceneTestCase {
ir.close();
dir.close();
}
/** Tests sorting on type string with a missing
* value sorted first */
public void testStringMissingSortedFirst() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "foo", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "bar", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sf = new SortField("value", SortField.Type.STRING);
Sort sort = new Sort(sf);
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// null comes first
assertNull(searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("foo", searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests reverse sorting on type string with a missing
* value sorted first */
public void testStringMissingSortedFirstReverse() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "foo", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "bar", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sf = new SortField("value", SortField.Type.STRING, true);
Sort sort = new Sort(sf);
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
assertEquals("foo", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value"));
// null comes last
assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests sorting on type string with a missing
* value sorted last */
public void testStringValMissingSortedLast() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "foo", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "bar", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sf = new SortField("value", SortField.Type.STRING);
sf.setMissingValue(SortField.STRING_LAST);
Sort sort = new Sort(sf);
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value"));
// null comes last
assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests reverse sorting on type string with a missing
* value sorted last */
public void testStringValMissingSortedLastReverse() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "foo", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "bar", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sf = new SortField("value", SortField.Type.STRING, true);
sf.setMissingValue(SortField.STRING_LAST);
Sort sort = new Sort(sf);
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// null comes first
assertNull(searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("bar", searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests reverse sorting on type string_val */
public void testStringValReverse() throws IOException {

128
lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
@ -29,6 +30,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
@ -60,30 +62,44 @@ public class TestSortRandom extends LuceneTestCase {
final List<BytesRef> docValues = new ArrayList<BytesRef>();
// TODO: deletions
while (numDocs < NUM_DOCS) {
final String s;
if (random.nextBoolean()) {
s = _TestUtil.randomSimpleString(random, maxLength);
} else {
s = _TestUtil.randomUnicodeString(random, maxLength);
}
final BytesRef br = new BytesRef(s);
if (!allowDups) {
if (seen.contains(s)) {
continue;
}
seen.add(s);
}
if (VERBOSE) {
System.out.println(" " + numDocs + ": s=" + s);
}
final Document doc = new Document();
doc.add(new SortedDocValuesField("stringdv", br));
doc.add(newStringField("string", s, Field.Store.NO));
// 10% of the time, the document is missing the value:
final BytesRef br;
if (random().nextInt(10) != 7) {
final String s;
if (random.nextBoolean()) {
s = _TestUtil.randomSimpleString(random, maxLength);
} else {
s = _TestUtil.randomUnicodeString(random, maxLength);
}
if (!allowDups) {
if (seen.contains(s)) {
continue;
}
seen.add(s);
}
if (VERBOSE) {
System.out.println(" " + numDocs + ": s=" + s);
}
br = new BytesRef(s);
doc.add(new SortedDocValuesField("stringdv", br));
doc.add(newStringField("string", s, Field.Store.NO));
docValues.add(br);
} else {
br = null;
if (VERBOSE) {
System.out.println(" " + numDocs + ": <missing>");
}
docValues.add(null);
}
doc.add(new NumericDocValuesField("id", numDocs));
docValues.add(br);
doc.add(new StoredField("id", numDocs));
writer.addDocument(doc);
numDocs++;
@ -103,13 +119,26 @@ public class TestSortRandom extends LuceneTestCase {
final int ITERS = atLeast(100);
for(int iter=0;iter<ITERS;iter++) {
final boolean reverse = random.nextBoolean();
final TopFieldDocs hits;
final SortField sf;
final boolean sortMissingLast;
final boolean missingIsNull;
if (random.nextBoolean()) {
sf = new SortField("stringdv", SortField.Type.STRING, reverse);
// Can only use sort missing if the DVFormat
// supports docsWithField:
sortMissingLast = defaultCodecSupportsDocsWithField() && random().nextBoolean();
missingIsNull = defaultCodecSupportsDocsWithField();
} else {
sf = new SortField("string", SortField.Type.STRING, reverse);
sortMissingLast = random().nextBoolean();
missingIsNull = true;
}
if (sortMissingLast) {
sf.setMissingValue(SortField.STRING_LAST);
}
final Sort sort;
if (random.nextBoolean()) {
sort = new Sort(sf);
@ -138,11 +167,34 @@ public class TestSortRandom extends LuceneTestCase {
}
if (VERBOSE) {
System.out.println("\nTEST: iter=" + iter + " " + hits.totalHits + " hits; topN=" + hitCount + "; reverse=" + reverse);
System.out.println("\nTEST: iter=" + iter + " " + hits.totalHits + " hits; topN=" + hitCount + "; reverse=" + reverse + "; sortMissingLast=" + sortMissingLast + " sort=" + sort);
}
// Compute expected results:
Collections.sort(f.matchValues);
Collections.sort(f.matchValues, new Comparator<BytesRef>() {
@Override
public int compare(BytesRef a, BytesRef b) {
if (a == null) {
if (b == null) {
return 0;
}
if (sortMissingLast) {
return 1;
} else {
return -1;
}
} else if (b == null) {
if (sortMissingLast) {
return -1;
} else {
return 1;
}
} else {
return a.compareTo(b);
}
}
});
if (reverse) {
Collections.reverse(f.matchValues);
}
@ -150,7 +202,11 @@ public class TestSortRandom extends LuceneTestCase {
if (VERBOSE) {
System.out.println(" expected:");
for(int idx=0;idx<expected.size();idx++) {
System.out.println(" " + idx + ": " + expected.get(idx).utf8ToString());
BytesRef br = expected.get(idx);
if (br == null && missingIsNull == false) {
br = new BytesRef();
}
System.out.println(" " + idx + ": " + (br == null ? "<missing>" : br.utf8ToString()));
if (idx == hitCount-1) {
break;
}
@ -161,12 +217,30 @@ public class TestSortRandom extends LuceneTestCase {
System.out.println(" actual:");
for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
System.out.println(" " + hitIDX + ": " + ((BytesRef) fd.fields[0]).utf8ToString());
BytesRef br = (BytesRef) fd.fields[0];
System.out.println(" " + hitIDX + ": " + (br == null ? "<missing>" : br.utf8ToString()) + " id=" + s.doc(fd.doc).get("id"));
}
}
for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
assertEquals(expected.get(hitIDX), (BytesRef) fd.fields[0]);
BytesRef br = expected.get(hitIDX);
if (br == null && missingIsNull == false) {
br = new BytesRef();
}
// Normally, the old codecs (that don't support
// docsWithField via doc values) will always return
// an empty BytesRef for the missing case; however,
// if all docs in a given segment were missing, in
// that case it will return null! So we must map
// null here, too:
BytesRef br2 = (BytesRef) fd.fields[0];
if (br2 == null && missingIsNull == false) {
br2 = new BytesRef();
}
assertEquals(br, br2);
}
}

10
lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionComparator.java
View File

@ -30,6 +30,7 @@ import org.apache.lucene.search.Scorer;
class ExpressionComparator extends FieldComparator<Double> {
private final double[] values;
private double bottom;
private double topValue;
private ValueSource source;
private FunctionValues scores;
@ -66,6 +67,11 @@ class ExpressionComparator extends FieldComparator<Double> {
bottom = values[slot];
}
@Override
public void setTopValue(Double value) {
topValue = value.doubleValue();
}
@Override
public int compareBottom(int doc) throws IOException {
return Double.compare(bottom, scores.doubleVal(doc));
@ -88,7 +94,7 @@ class ExpressionComparator extends FieldComparator<Double> {
}
@Override
public int compareDocToValue(int doc, Double valueObj) throws IOException {
return Double.compare(scores.doubleVal(doc), valueObj.doubleValue());
public int compareTop(int doc) throws IOException {
return Double.compare(topValue, scores.doubleVal(doc));
}
}

13
lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinFieldComparator.java
View File

@ -59,6 +59,11 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
wrappedComparator.setBottom(slot);
}
@Override
public void setTopValue(Object value) {
wrappedComparator.setTopValue(value);
}
@Override
public FieldComparator<Object> setNextReader(AtomicReaderContext context) throws IOException {
DocIdSet innerDocuments = childFilter.getDocIdSet(context, null);
@ -193,7 +198,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
@Override
@SuppressWarnings("unchecked")
public int compareDocToValue(int parentDoc, Object value) throws IOException {
public int compareTop(int parentDoc) throws IOException {
if (parentDoc == 0 || parentDocuments == null || childDocuments == null) {
return 0;
}
@ -216,7 +221,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
if (childDoc >= parentDoc || childDoc == -1) {
return cmp;
}
int cmp1 = wrappedComparator.compareDocToValue(childDoc, value);
int cmp1 = wrappedComparator.compareTop(childDoc);
if (cmp1 > 0) {
return cmp1;
} else {
@ -309,7 +314,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
@Override
@SuppressWarnings("unchecked")
public int compareDocToValue(int parentDoc, Object value) throws IOException {
public int compareTop(int parentDoc) throws IOException {
if (parentDoc == 0 || parentDocuments == null || childDocuments == null) {
return 0;
}
@ -330,7 +335,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator<O
if (childDoc >= parentDoc || childDoc == -1) {
return cmp;
}
int cmp1 = wrappedComparator.compareDocToValue(childDoc, value);
int cmp1 = wrappedComparator.compareTop(childDoc);
if (cmp1 < 0) {
return cmp1;
} else {

11
lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
View File

@ -131,6 +131,7 @@ public abstract class ValueSource {
private FunctionValues docVals;
private double bottom;
private final Map fcontext;
private double topValue;
ValueSourceComparator(Map fcontext, int numHits) {
this.fcontext = fcontext;
@ -163,16 +164,20 @@ public abstract class ValueSource {
this.bottom = values[bottom];
}
@Override
public void setTopValue(final Double value) {
this.topValue = value.doubleValue();
}
@Override
public Double value(int slot) {
return values[slot];
}
@Override
public int compareDocToValue(int doc, Double valueObj) {
final double value = valueObj;
public int compareTop(int doc) {
final double docValue = docVals.doubleVal(doc);
return Double.compare(docValue, value);
return Double.compare(topValue, docValue);
}
}
}

10
lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java
View File

@ -44,6 +44,7 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
private final String field;
final Collator collator;
private String bottom;
private String topValue;
private final BytesRef tempBR = new BytesRef();
public SlowCollatedStringComparator(int numHits, String field, Collator collator) {
@ -104,6 +105,11 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
this.bottom = values[bottom];
}
@Override
public void setTopValue(final String value) {
this.topValue = value;
}
@Override
public String value(int slot) {
return values[slot];
@ -124,7 +130,7 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
}
@Override
public int compareDocToValue(int doc, String value) {
public int compareTop(int doc) {
currentDocTerms.get(doc, tempBR);
final String docValue;
if (tempBR.length == 0 && docsWithField.get(doc) == false) {
@ -132,6 +138,6 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
} else {
docValue = tempBR.utf8ToString();
}
return compareValues(docValue, value);
return compareValues(topValue, docValue);
}
}

2
lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
View File

@ -632,7 +632,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
randomIOExceptionRateOnOpen = 0.0;
if (DirectoryReader.indexExists(this)) {
if (LuceneTestCase.VERBOSE) {
System.out.println("\nNOTE: MockDirectoryWrapper: now crash");
System.out.println("\nNOTE: MockDirectoryWrapper: now crush");
}
crash(); // corrupt any unsynced-files
if (LuceneTestCase.VERBOSE) {

4
solr/CHANGES.txt
View File

@ -209,6 +209,10 @@ Bug Fixes
* SOLR-5636: SolrRequestParsers does some xpath lookups on every request, which
can cause concurrency issues. (Mark Miller)
* LUCENE-5399, SOLR-5354 sort wouldn't work correctly with
distributed searching for some field types such as legacy numeric
types (Rob Muir, Mike McCandless)
Optimizations
----------------------

11
solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
View File

@ -576,6 +576,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
return new FieldComparator<Integer>() {
private final int[] values = new int[numHits];
private int bottomVal;
private int topVal;
private TermsEnum termsEnum;
private DocsEnum docsEnum;
Set<String> seen = new HashSet<String>(elevations.ids.size());
@ -590,6 +591,11 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
bottomVal = values[slot];
}
@Override
public void setTopValue(Integer value) {
topVal = value.intValue();
}
private int docVal(int doc) {
if (ordSet.size() > 0) {
int slot = ordSet.find(doc);
@ -646,10 +652,9 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
}
@Override
public int compareDocToValue(int doc, Integer valueObj) {
final int value = valueObj.intValue();
public int compareTop(int doc) {
final int docValue = docVal(doc);
return docValue - value; // values will be small enough that there is no overflow concern
return topVal - docValue; // values will be small enough that there is no overflow concern
}
};
}

4
solr/core/src/java/org/apache/solr/schema/EnumField.java
View File

@ -178,7 +178,9 @@ public class EnumField extends PrimitiveFieldType {
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
final Object missingValue = Integer.MIN_VALUE;
return new SortField(field.getName(), FieldCache.NUMERIC_UTILS_INT_PARSER, top).setMissingValue(missingValue);
SortField sf = new SortField(field.getName(), FieldCache.NUMERIC_UTILS_INT_PARSER, top);
sf.setMissingValue(missingValue);
return sf;
}
/**

10
solr/core/src/java/org/apache/solr/schema/RandomSortField.java
View File

@ -109,6 +109,7 @@ public class RandomSortField extends FieldType {
int seed;
private final int[] values = new int[numHits];
int bottomVal;
int topVal;
@Override
public int compare(int slot1, int slot2) {
@ -120,6 +121,11 @@ public class RandomSortField extends FieldType {
bottomVal = values[slot];
}
@Override
public void setTopValue(Integer value) {
topVal = value.intValue();
}
@Override
public int compareBottom(int doc) {
return bottomVal - hash(doc+seed);
@ -142,9 +148,9 @@ public class RandomSortField extends FieldType {
}
@Override
public int compareDocToValue(int doc, Integer valueObj) {
public int compareTop(int doc) {
// values will be positive... no overflow possible.
return hash(doc+seed) - valueObj.intValue();
return topVal - hash(doc+seed);
}
};
}

20
solr/core/src/java/org/apache/solr/schema/TrieField.java
View File

@ -142,7 +142,9 @@ public class TrieField extends PrimitiveFieldType {
Object missingValue = null;
boolean sortMissingLast = field.sortMissingLast();
boolean sortMissingFirst = field.sortMissingFirst();
SortField sf;
switch (type) {
case INTEGER:
if( sortMissingLast ) {
@ -151,7 +153,9 @@ public class TrieField extends PrimitiveFieldType {
else if( sortMissingFirst ) {
missingValue = top ? Integer.MAX_VALUE : Integer.MIN_VALUE;
}
return new SortField( field.getName(), FieldCache.NUMERIC_UTILS_INT_PARSER, top).setMissingValue(missingValue);
sf = new SortField( field.getName(), FieldCache.NUMERIC_UTILS_INT_PARSER, top);
sf.setMissingValue(missingValue);
return sf;
case FLOAT:
if( sortMissingLast ) {
@ -160,7 +164,9 @@ public class TrieField extends PrimitiveFieldType {
else if( sortMissingFirst ) {
missingValue = top ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY;
}
return new SortField( field.getName(), FieldCache.NUMERIC_UTILS_FLOAT_PARSER, top).setMissingValue(missingValue);
sf = new SortField( field.getName(), FieldCache.NUMERIC_UTILS_FLOAT_PARSER, top);
sf.setMissingValue(missingValue);
return sf;
case DATE: // fallthrough
case LONG:
@ -170,7 +176,9 @@ public class TrieField extends PrimitiveFieldType {
else if( sortMissingFirst ) {
missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
}
return new SortField( field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, top).setMissingValue(missingValue);
sf = new SortField( field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, top);
sf.setMissingValue(missingValue);
return sf;
case DOUBLE:
if( sortMissingLast ) {
@ -179,7 +187,9 @@ public class TrieField extends PrimitiveFieldType {
else if( sortMissingFirst ) {
missingValue = top ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
}
return new SortField( field.getName(), FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, top).setMissingValue(missingValue);
sf = new SortField( field.getName(), FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, top);
sf.setMissingValue(missingValue);
return sf;
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + field.name);

310
solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java
View File

@ -1,310 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import java.io.IOException;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.FieldComparatorSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.packed.PackedInts;
public class MissingStringLastComparatorSource extends FieldComparatorSource {
private final BytesRef missingValueProxy;
public MissingStringLastComparatorSource() {
this(UnicodeUtil.BIG_TERM);
}
/** Creates a {@link FieldComparatorSource} that sorts null last in a normal ascending sort.
* <tt>missingValueProxy</tt> as the value to return from FieldComparator.value()
*
* @param missingValueProxy The value returned when sortValue() is called for a document missing the sort field.
* This value is *not* normally used for sorting.
*/
public MissingStringLastComparatorSource(BytesRef missingValueProxy) {
this.missingValueProxy=missingValueProxy;
}
@Override
public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
return new TermOrdValComparator_SML(numHits, fieldname, sortPos, reversed, missingValueProxy);
}
}
// Copied from Lucene's TermOrdValComparator and modified since the Lucene version couldn't
// be extended.
class TermOrdValComparator_SML extends FieldComparator<Comparable> {
private static final int NULL_ORD = Integer.MAX_VALUE-1;
private final int[] ords;
private final BytesRef[] values;
private final int[] readerGen;
private SortedDocValues termsIndex;
private final String field;
private final BytesRef NULL_VAL;
private PerSegmentComparator current;
public TermOrdValComparator_SML(int numHits, String field, int sortPos, boolean reversed, BytesRef nullVal) {
ords = new int[numHits];
values = new BytesRef[numHits];
readerGen = new int[numHits];
this.field = field;
this.NULL_VAL = nullVal;
}
@Override
public int compare(int slot1, int slot2) {
throw new UnsupportedOperationException();
}
@Override
public void setBottom(int slot) {
throw new UnsupportedOperationException();
}
@Override
public int compareBottom(int doc) {
throw new UnsupportedOperationException();
}
@Override
public void copy(int slot, int doc) {
throw new UnsupportedOperationException();
}
@Override
public BytesRef value(int slot) {
throw new UnsupportedOperationException();
}
@Override
public int compareValues(Comparable first, Comparable second) {
if (first == null) {
if (second == null) {
return 0;
} else {
return 1;
}
} else if (second == null) {
return -1;
} else {
return first.compareTo(second);
}
}
@Override
public FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
return TermOrdValComparator_SML.createComparator(context.reader(), this);
}
@Override
public int compareDocToValue(int doc, Comparable docValue) {
throw new UnsupportedOperationException();
}
// Base class for specialized (per bit width of the
// ords) per-segment comparator. NOTE: this is messy;
// we do this only because hotspot can't reliably inline
// the underlying array access when looking up doc->ord
private static abstract class PerSegmentComparator extends FieldComparator<BytesRef> {
protected TermOrdValComparator_SML parent;
protected final int[] ords;
protected final BytesRef[] values;
protected final int[] readerGen;
protected int currentReaderGen = -1;
protected SortedDocValues termsIndex;
protected int bottomSlot = -1;
protected int bottomOrd;
protected boolean bottomSameReader = false;
protected BytesRef bottomValue;
protected final BytesRef tempBR = new BytesRef();
public PerSegmentComparator(TermOrdValComparator_SML parent) {
this.parent = parent;
PerSegmentComparator previous = parent.current;
if (previous != null) {
currentReaderGen = previous.currentReaderGen;
bottomSlot = previous.bottomSlot;
bottomOrd = previous.bottomOrd;
bottomValue = previous.bottomValue;
}
ords = parent.ords;
values = parent.values;
readerGen = parent.readerGen;
termsIndex = parent.termsIndex;
currentReaderGen++;
}
@Override
public FieldComparator setNextReader(AtomicReaderContext context) throws IOException {
return TermOrdValComparator_SML.createComparator(context.reader(), parent);
}
@Override
public int compare(int slot1, int slot2) {
if (readerGen[slot1] == readerGen[slot2]) {
return ords[slot1] - ords[slot2];
}
final BytesRef val1 = values[slot1];
final BytesRef val2 = values[slot2];
if (val1 == null) {
if (val2 == null) {
return 0;
}
return 1;
} else if (val2 == null) {
return -1;
}
return val1.compareTo(val2);
}
@Override
public void setBottom(final int bottom) {
bottomSlot = bottom;
bottomValue = values[bottomSlot];
if (currentReaderGen == readerGen[bottomSlot]) {
bottomOrd = ords[bottomSlot];
bottomSameReader = true;
} else {
if (bottomValue == null) {
// -1 ord is null for all segments
assert ords[bottomSlot] == NULL_ORD;
bottomOrd = NULL_ORD;
bottomSameReader = true;
readerGen[bottomSlot] = currentReaderGen;
} else {
final int index = termsIndex.lookupTerm(bottomValue);
if (index < 0) {
bottomOrd = -index - 2;
bottomSameReader = false;
} else {
bottomOrd = index;
// exact value match
bottomSameReader = true;
readerGen[bottomSlot] = currentReaderGen;
ords[bottomSlot] = bottomOrd;
}
}
}
}
@Override
public BytesRef value(int slot) {
return values==null ? parent.NULL_VAL : values[slot];
}
@Override
public int compareDocToValue(int doc, BytesRef value) {
int docOrd = termsIndex.getOrd(doc);
if (docOrd == -1) {
if (value == null) {
return 0;
}
return 1;
} else if (value == null) {
return -1;
}
termsIndex.lookupOrd(docOrd, tempBR);
return tempBR.compareTo(value);
}
}
private static final class AnyOrdComparator extends PerSegmentComparator {
public AnyOrdComparator(TermOrdValComparator_SML parent) {
super(parent);
}
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
int order = termsIndex.getOrd(doc);
if (order == -1) order = NULL_ORD;
if (bottomSameReader) {
// ord is precisely comparable, even in the equal
// case
return bottomOrd - order;
} else {
// ord is only approx comparable: if they are not
// equal, we can use that; if they are equal, we
// must fallback to compare by value
final int cmp = bottomOrd - order;
if (cmp != 0) {
return cmp;
}
// take care of the case where both vals are null
if (order == NULL_ORD) {
return 0;
}
// and at this point we know that neither value is null, so safe to compare
if (order == NULL_ORD) {
return bottomValue.compareTo(parent.NULL_VAL);
} else {
termsIndex.lookupOrd(order, tempBR);
return bottomValue.compareTo(tempBR);
}
}
}
@Override
public void copy(int slot, int doc) {
int ord = termsIndex.getOrd(doc);
if (ord == -1) {
ords[slot] = NULL_ORD;
values[slot] = null;
} else {
ords[slot] = ord;
assert ord >= 0;
if (values[slot] == null) {
values[slot] = new BytesRef();
}
termsIndex.lookupOrd(ord, values[slot]);
}
readerGen[slot] = currentReaderGen;
}
}
public static FieldComparator createComparator(AtomicReader reader, TermOrdValComparator_SML parent) throws IOException {
parent.termsIndex = FieldCache.DEFAULT.getTermsIndex(reader, parent.field);
PerSegmentComparator perSegComp = new AnyOrdComparator(parent);
if (perSegComp.bottomSlot != -1) {
perSegComp.setBottom(perSegComp.bottomSlot);
}
parent.current = perSegComp;
return perSegComp;
}
}

29
solr/core/src/java/org/apache/solr/search/Sorting.java
View File

@ -40,18 +40,25 @@ public class Sorting {
* @return SortField
*/
public static SortField getStringSortField(String fieldName, boolean reverse, boolean nullLast, boolean nullFirst) {
if (nullLast) {
if (!reverse) return new SortField(fieldName, nullStringLastComparatorSource);
else return new SortField(fieldName, SortField.Type.STRING, true);
} else if (nullFirst) {
if (reverse) return new SortField(fieldName, nullStringLastComparatorSource, true);
else return new SortField(fieldName, SortField.Type.STRING, false);
} else {
return new SortField(fieldName, SortField.Type.STRING, reverse);
if (nullFirst && nullLast) {
throw new IllegalArgumentException("Cannot specify missing values as both first and last");
}
SortField sortField = new SortField(fieldName, SortField.Type.STRING, reverse);
// 4 cases:
// missingFirst / forward: default lucene behavior
// missingFirst / reverse: set sortMissingLast
// missingLast / forward: set sortMissingLast
// missingLast / reverse: default lucene behavior
if (nullFirst && reverse) {
sortField.setMissingValue(SortField.STRING_LAST);
} else if (nullLast && !reverse) {
sortField.setMissingValue(SortField.STRING_LAST);
}
return sortField;
}
static final FieldComparatorSource nullStringLastComparatorSource = new MissingStringLastComparatorSource(null);
}

24
solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java
View File

@ -88,6 +88,15 @@ public class SearchGroupsResultTransformer implements ShardResultTransformer<Lis
SearchGroup<BytesRef> searchGroup = new SearchGroup<BytesRef>();
searchGroup.groupValue = rawSearchGroup.getKey() != null ? new BytesRef(rawSearchGroup.getKey()) : null;
searchGroup.sortValues = rawSearchGroup.getValue().toArray(new Comparable[rawSearchGroup.getValue().size()]);
for (int i = 0; i < searchGroup.sortValues.length; i++) {
SchemaField field = groupSort.getSort()[i].getField() != null ? searcher.getSchema().getFieldOrNull(groupSort.getSort()[i].getField()) : null;
if (field != null) {
FieldType fieldType = field.getType();
if (searchGroup.sortValues[i] != null) {
searchGroup.sortValues[i] = fieldType.unmarshalSortValue(searchGroup.sortValues[i]);
}
}
}
searchGroups.add(searchGroup);
}
}
@ -99,22 +108,17 @@ public class SearchGroupsResultTransformer implements ShardResultTransformer<Lis
}
private NamedList serializeSearchGroup(Collection<SearchGroup<BytesRef>> data, Sort groupSort) {
NamedList<Comparable[]> result = new NamedList<Comparable[]>();
CharsRef spare = new CharsRef();
NamedList<Object[]> result = new NamedList<Object[]>();
for (SearchGroup<BytesRef> searchGroup : data) {
Comparable[] convertedSortValues = new Comparable[searchGroup.sortValues.length];
Object[] convertedSortValues = new Object[searchGroup.sortValues.length];
for (int i = 0; i < searchGroup.sortValues.length; i++) {
Comparable sortValue = (Comparable) searchGroup.sortValues[i];
Object sortValue = searchGroup.sortValues[i];
SchemaField field = groupSort.getSort()[i].getField() != null ? searcher.getSchema().getFieldOrNull(groupSort.getSort()[i].getField()) : null;
if (field != null) {
FieldType fieldType = field.getType();
if (sortValue instanceof BytesRef) {
UnicodeUtil.UTF8toUTF16((BytesRef)sortValue, spare);
String indexedValue = spare.toString();
sortValue = (Comparable) fieldType.toObject(field.createField(fieldType.indexedToReadable(indexedValue), 1.0f));
} else if (sortValue instanceof String) {
sortValue = (Comparable) fieldType.toObject(field.createField(fieldType.indexedToReadable((String) sortValue), 1.0f));
if (sortValue != null) {
sortValue = fieldType.marshalSortValue(sortValue);
}
}
convertedSortValues[i] = sortValue;

36
solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
View File

@ -93,6 +93,8 @@ public class TopGroupsResultTransformer implements ShardResultTransformer<List<C
public Map<String, ?> transformToNative(NamedList<NamedList> shardResponse, Sort groupSort, Sort sortWithinGroup, String shard) {
Map<String, Object> result = new HashMap<String, Object>();
final IndexSchema schema = rb.req.getSearcher().getSchema();
for (Map.Entry<String, NamedList> entry : shardResponse) {
String key = entry.getKey();
NamedList commandResult = entry.getValue();
@ -124,6 +126,15 @@ public class TopGroupsResultTransformer implements ShardResultTransformer<List<C
Object sortValuesVal = document.get("sortValues");
if (sortValuesVal != null) {
sortValues = ((List) sortValuesVal).toArray();
for (int k = 0; k < sortValues.length; k++) {
SchemaField field = groupSort.getSort()[k].getField() != null ? schema.getFieldOrNull(groupSort.getSort()[k].getField()) : null;
if (field != null) {
FieldType fieldType = field.getType();
if (sortValues[k] != null) {
sortValues[k] = fieldType.unmarshalSortValue(sortValues[k]);
}
}
}
}
else {
log.warn("doc {} has null 'sortValues'", document);
@ -158,6 +169,15 @@ public class TopGroupsResultTransformer implements ShardResultTransformer<List<C
score = Float.NaN;
}
Object[] sortValues = ((List) document.get("sortValues")).toArray();
for (int k = 0; k < sortValues.length; k++) {
SchemaField field = sortWithinGroup.getSort()[k].getField() != null ? schema.getFieldOrNull(sortWithinGroup.getSort()[k].getField()) : null;
if (field != null) {
FieldType fieldType = field.getType();
if (sortValues[k] != null) {
sortValues[k] = fieldType.unmarshalSortValue(sortValues[k]);
}
}
}
scoreDocs[j++] = new ShardDoc(score, sortValues, uniqueId, shard);
}
@ -217,12 +237,8 @@ public class TopGroupsResultTransformer implements ShardResultTransformer<List<C
SchemaField field = sortWithinGroup.getSort()[j].getField() != null ? schema.getFieldOrNull(sortWithinGroup.getSort()[j].getField()) : null;
if (field != null) {
FieldType fieldType = field.getType();
if (sortValue instanceof BytesRef) {
UnicodeUtil.UTF8toUTF16((BytesRef)sortValue, spare);
String indexedValue = spare.toString();
sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable(indexedValue), 1.0f));
} else if (sortValue instanceof String) {
sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable((String) sortValue), 1.0f));
if (sortValue != null) {
sortValue = fieldType.marshalSortValue(sortValue);
}
}
convertedSortValues[j] = sortValue;
@ -272,12 +288,8 @@ public class TopGroupsResultTransformer implements ShardResultTransformer<List<C
? schema.getFieldOrNull(groupSort.getSort()[j].getField()) : null;
if (field != null) {
FieldType fieldType = field.getType();
if (sortValue instanceof BytesRef) {
UnicodeUtil.UTF8toUTF16((BytesRef)sortValue, spare);
String indexedValue = spare.toString();
sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable(indexedValue), 1.0f));
} else if (sortValue instanceof String) {
sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable((String) sortValue), 1.0f));
if (sortValue != null) {
sortValue = fieldType.marshalSortValue(sortValue);
}
}
convertedSortValues[j] = sortValue;