Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5166: clear most nocommits, move ord/rord to solr (and speed them up), nuke old purging stuff 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene5666@1594464 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-05-14 05:51:24 +00:00
parent b3a556bb32
commit 7b63afc003
47 changed files with 1017 additions and 1086 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/analysis/common/src/java/org/apache/lucene/collation/CollationDocValuesField.java
View File

@ -21,7 +21,7 @@ import java.text.Collator;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.DocValuesRangeFilter;
import org.apache.lucene.util.BytesRef;
/**
@ -29,7 +29,7 @@ import org.apache.lucene.util.BytesRef;
* <p>
* This is more efficient that {@link CollationKeyAnalyzer} if the field
* only has one value: no uninversion is necessary to sort on the field,
* locale-sensitive range queries can still work via {@link FieldCacheRangeFilter},
* locale-sensitive range queries can still work via {@link DocValuesRangeFilter},
* and the underlying data structures built at index-time are likely more efficient
* and use less memory than FieldCache.
*/

4
lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java
View File

@ -27,7 +27,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.DocValuesRangeFilter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
@ -111,7 +111,7 @@ public class TestCollationDocValuesField extends LuceneTestCase {
String end = TestUtil.randomSimpleString(random());
BytesRef lowerVal = new BytesRef(collator.getCollationKey(start).toByteArray());
BytesRef upperVal = new BytesRef(collator.getCollationKey(end).toByteArray());
Query query = new ConstantScoreQuery(FieldCacheRangeFilter.newBytesRefRange("collated", lowerVal, upperVal, true, true));
Query query = new ConstantScoreQuery(DocValuesRangeFilter.newBytesRefRange("collated", lowerVal, upperVal, true, true));
doTestRanges(is, start, end, query, collator);
}

4
lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationDocValuesField.java
View File

@ -19,7 +19,7 @@ package org.apache.lucene.collation;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.DocValuesRangeFilter;
import org.apache.lucene.util.BytesRef;
import com.ibm.icu.text.Collator;
@ -30,7 +30,7 @@ import com.ibm.icu.text.RawCollationKey;
* <p>
* This is more efficient that {@link ICUCollationKeyAnalyzer} if the field
* only has one value: no uninversion is necessary to sort on the field,
* locale-sensitive range queries can still work via {@link FieldCacheRangeFilter},
* locale-sensitive range queries can still work via {@link DocValuesRangeFilter},
* and the underlying data structures built at index-time are likely more efficient
* and use less memory than FieldCache.
*/

4
lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationDocValuesField.java
View File

@ -24,7 +24,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.DocValuesRangeFilter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
@ -111,7 +111,7 @@ public class TestICUCollationDocValuesField extends LuceneTestCase {
String end = TestUtil.randomSimpleString(random());
BytesRef lowerVal = new BytesRef(collator.getCollationKey(start).toByteArray());
BytesRef upperVal = new BytesRef(collator.getCollationKey(end).toByteArray());
Query query = new ConstantScoreQuery(FieldCacheRangeFilter.newBytesRefRange("collated", lowerVal, upperVal, true, true));
Query query = new ConstantScoreQuery(DocValuesRangeFilter.newBytesRefRange("collated", lowerVal, upperVal, true, true));
doTestRanges(is, start, end, query, collator);
}

4
lucene/core/src/java/org/apache/lucene/index/IndexReader.java
View File

@ -426,7 +426,7 @@ public abstract class IndexReader implements Closeable {
return getContext().leaves();
}
/** Expert: Returns a key for this IndexReader, so FieldCache/CachingWrapperFilter can find
/** Expert: Returns a key for this IndexReader, so CachingWrapperFilter can find
* it again.
* This key must not have equals()/hashCode() methods, so &quot;equals&quot; means &quot;identical&quot;. */
public Object getCoreCacheKey() {
@ -436,7 +436,7 @@ public abstract class IndexReader implements Closeable {
}
/** Expert: Returns a key for this IndexReader that also includes deletions,
* so FieldCache/CachingWrapperFilter can find it again.
* so CachingWrapperFilter can find it again.
* This key must not have equals()/hashCode() methods, so &quot;equals&quot; means &quot;identical&quot;. */
public Object getCombinedCoreAndDeletesKey() {
// Don't call ensureOpen since FC calls this (to evict)

2
lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
View File

@ -361,7 +361,7 @@ public final class SegmentReader extends AtomicReader {
// This is necessary so that cloned SegmentReaders (which
// share the underlying postings data) will map to the
// same entry in the FieldCache. See LUCENE-1579.
// same entry for CachingWrapperFilter. See LUCENE-1579.
@Override
public Object getCoreCacheKey() {
// NOTE: if this ever changes, be sure to fix

3
lucene/core/src/java/org/apache/lucene/index/SingletonSortedSetDocValues.java
View File

@ -23,8 +23,7 @@ import org.apache.lucene.util.BytesRef;
* Exposes multi-valued view over a single-valued instance.
* <p>
* This can be used if you want to have one multi-valued implementation
* against e.g. FieldCache.getDocTermOrds that also works for single-valued
* fields.
* that works for single or multi-valued types.
*/
final class SingletonSortedSetDocValues extends SortedSetDocValues {
private final SortedDocValues in;

4
lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRangeFilter.java
View File

@ -28,7 +28,7 @@ import org.apache.lucene.util.BytesRef;
/**
* A range filter built on top of a cached multi-valued term field (from {@link AtomicReader#getSortedSetDocValues}).
*
* <p>Like {@link FieldCacheRangeFilter}, this is just a specialized range query versus
* <p>Like {@link DocValuesRangeFilter}, this is just a specialized range query versus
* using a TermRangeQuery with {@link DocTermOrdsRewriteMethod}: it will only do
* two ordinal to term lookups.</p>
*/
@ -97,7 +97,7 @@ public abstract class DocTermOrdsRangeFilter extends Filter {
assert inclusiveLowerPoint >= 0 && inclusiveUpperPoint >= 0;
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected final boolean matchDoc(int doc) {
docTermOrds.setDocument(doc);

2
lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRewriteMethod.java
View File

@ -145,7 +145,7 @@ public final class DocTermOrdsRewriteMethod extends MultiTermQuery.RewriteMethod
return null;
}
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected final boolean matchDoc(int doc) throws ArrayIndexOutOfBoundsException {
docTermOrds.setDocument(doc);

8
lucene/core/src/java/org/apache/lucene/search/FieldCacheDocIdSet.java → lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java
View File

@ -22,7 +22,7 @@ import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.OpenBitSet;
/**
* Base class for DocIdSet to be used with FieldCache. The implementation
* Base class for DocIdSet to be used with DocValues. The implementation
* of its iterator is very stupid and slow if the implementation of the
* {@link #matchDoc} method is not optimized, as iterators simply increment
* the document id until {@code matchDoc(int)} returns true. Because of this
@ -30,12 +30,12 @@ import org.apache.lucene.util.OpenBitSet;
* I/O.
* @lucene.internal
*/
public abstract class FieldCacheDocIdSet extends DocIdSet {
public abstract class DocValuesDocIdSet extends DocIdSet {
protected final int maxDoc;
protected final Bits acceptDocs;
public FieldCacheDocIdSet(int maxDoc, Bits acceptDocs) {
public DocValuesDocIdSet(int maxDoc, Bits acceptDocs) {
this.maxDoc = maxDoc;
this.acceptDocs = acceptDocs;
}
@ -123,7 +123,7 @@ public abstract class FieldCacheDocIdSet extends DocIdSet {
return new FilteredDocIdSetIterator(((DocIdSet) acceptDocs).iterator()) {
@Override
protected boolean match(int doc) {
return FieldCacheDocIdSet.this.matchDoc(doc);
return DocValuesDocIdSet.this.matchDoc(doc);
}
};
} else {

53
lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java → lucene/core/src/java/org/apache/lucene/search/DocValuesRangeFilter.java
View File

@ -35,11 +35,11 @@ import org.apache.lucene.util.NumericUtils;
* A range filter built on top of an uninverted single term field
* (from {@link AtomicReader#getNumericDocValues(String)}).
*
* <p>{@code FieldCacheRangeFilter} builds a single cache for the field the first time it is used.
* Each subsequent {@code FieldCacheRangeFilter} on the same field then reuses this cache,
* <p>{@code DocValuesRangeFilter} builds a single cache for the field the first time it is used.
* Each subsequent {@code DocValuesRangeFilter} on the same field then reuses this cache,
* even if the range itself changes.
*
* <p>This means that {@code FieldCacheRangeFilter} is much faster (sometimes more than 100x as fast)
* <p>This means that {@code DocValuesRangeFilter} is much faster (sometimes more than 100x as fast)
* as building a {@link TermRangeFilter}, if using a {@link #newStringRange}.
* However, if the range never changes it is slower (around 2x as slow) than building
* a CachingWrapperFilter on top of a single {@link TermRangeFilter}.
@ -51,7 +51,7 @@ import org.apache.lucene.util.NumericUtils;
* it has the problem that it only works with exact one value/document (see below).
*
* <p>As with all {@link AtomicReader#getNumericDocValues} based functionality,
* {@code FieldCacheRangeFilter} is only valid for
* {@code DocValuesRangeFilter} is only valid for
* fields which exact one term for each document (except for {@link #newStringRange}
* where 0 terms are also allowed). Due to historical reasons, for numeric ranges
* all terms that do not have a numeric value, 0 is assumed.
@ -63,16 +63,15 @@ import org.apache.lucene.util.NumericUtils;
* <p>This class does not have an constructor, use one of the static factory methods available,
* that create a correct instance for different data types.
*/
// nocommit: rename this class
// TODO: use docsWithField to handle empty properly
public abstract class FieldCacheRangeFilter<T> extends Filter {
public abstract class DocValuesRangeFilter<T> extends Filter {
final String field;
final T lowerVal;
final T upperVal;
final boolean includeLower;
final boolean includeUpper;
private FieldCacheRangeFilter(String field, T lowerVal, T upperVal, boolean includeLower, boolean includeUpper) {
private DocValuesRangeFilter(String field, T lowerVal, T upperVal, boolean includeLower, boolean includeUpper) {
this.field = field;
this.lowerVal = lowerVal;
this.upperVal = upperVal;
@ -89,8 +88,8 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
* fields containing zero or one term in the field. The range can be half-open by setting one
* of the values to <code>null</code>.
*/
public static FieldCacheRangeFilter<String> newStringRange(String field, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
return new FieldCacheRangeFilter<String>(field, lowerVal, upperVal, includeLower, includeUpper) {
public static DocValuesRangeFilter<String> newStringRange(String field, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
return new DocValuesRangeFilter<String>(field, lowerVal, upperVal, includeLower, includeUpper) {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final SortedDocValues fcsi = DocValues.getSorted(context.reader(), field);
@ -129,7 +128,7 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
assert inclusiveLowerPoint >= 0 && inclusiveUpperPoint >= 0;
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected final boolean matchDoc(int doc) {
final int docOrd = fcsi.getOrd(doc);
@ -146,8 +145,8 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
* of the values to <code>null</code>.
*/
// TODO: bogus that newStringRange doesnt share this code... generics hell
public static FieldCacheRangeFilter<BytesRef> newBytesRefRange(String field, BytesRef lowerVal, BytesRef upperVal, boolean includeLower, boolean includeUpper) {
return new FieldCacheRangeFilter<BytesRef>(field, lowerVal, upperVal, includeLower, includeUpper) {
public static DocValuesRangeFilter<BytesRef> newBytesRefRange(String field, BytesRef lowerVal, BytesRef upperVal, boolean includeLower, boolean includeUpper) {
return new DocValuesRangeFilter<BytesRef>(field, lowerVal, upperVal, includeLower, includeUpper) {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final SortedDocValues fcsi = DocValues.getSorted(context.reader(), field);
@ -186,7 +185,7 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
assert inclusiveLowerPoint >= 0 && inclusiveUpperPoint >= 0;
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected final boolean matchDoc(int doc) {
final int docOrd = fcsi.getOrd(doc);
@ -202,8 +201,8 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
* int fields containing exactly one numeric term in the field. The range can be half-open by setting one
* of the values to <code>null</code>.
*/
public static FieldCacheRangeFilter<Integer> newIntRange(String field, Integer lowerVal, Integer upperVal, boolean includeLower, boolean includeUpper) {
return new FieldCacheRangeFilter<Integer>(field, lowerVal, upperVal, includeLower, includeUpper) {
public static DocValuesRangeFilter<Integer> newIntRange(String field, Integer lowerVal, Integer upperVal, boolean includeLower, boolean includeUpper) {
return new DocValuesRangeFilter<Integer>(field, lowerVal, upperVal, includeLower, includeUpper) {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final int inclusiveLowerPoint, inclusiveUpperPoint;
@ -228,7 +227,7 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
return null;
final NumericDocValues values = DocValues.getNumeric(context.reader(), field);
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected boolean matchDoc(int doc) {
final int value = (int) values.get(doc);
@ -244,8 +243,8 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
* long fields containing exactly one numeric term in the field. The range can be half-open by setting one
* of the values to <code>null</code>.
*/
public static FieldCacheRangeFilter<Long> newLongRange(String field, Long lowerVal, Long upperVal, boolean includeLower, boolean includeUpper) {
return new FieldCacheRangeFilter<Long>(field, lowerVal, upperVal, includeLower, includeUpper) {
public static DocValuesRangeFilter<Long> newLongRange(String field, Long lowerVal, Long upperVal, boolean includeLower, boolean includeUpper) {
return new DocValuesRangeFilter<Long>(field, lowerVal, upperVal, includeLower, includeUpper) {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final long inclusiveLowerPoint, inclusiveUpperPoint;
@ -270,7 +269,7 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
return null;
final NumericDocValues values = DocValues.getNumeric(context.reader(), field);
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected boolean matchDoc(int doc) {
final long value = values.get(doc);
@ -286,8 +285,8 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
* float fields containing exactly one numeric term in the field. The range can be half-open by setting one
* of the values to <code>null</code>.
*/
public static FieldCacheRangeFilter<Float> newFloatRange(String field, Float lowerVal, Float upperVal, boolean includeLower, boolean includeUpper) {
return new FieldCacheRangeFilter<Float>(field, lowerVal, upperVal, includeLower, includeUpper) {
public static DocValuesRangeFilter<Float> newFloatRange(String field, Float lowerVal, Float upperVal, boolean includeLower, boolean includeUpper) {
return new DocValuesRangeFilter<Float>(field, lowerVal, upperVal, includeLower, includeUpper) {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
// we transform the floating point numbers to sortable integers
@ -316,7 +315,7 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
return null;
final NumericDocValues values = DocValues.getNumeric(context.reader(), field);
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected boolean matchDoc(int doc) {
final float value = Float.intBitsToFloat((int)values.get(doc));
@ -332,8 +331,8 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
* double fields containing exactly one numeric term in the field. The range can be half-open by setting one
* of the values to <code>null</code>.
*/
public static FieldCacheRangeFilter<Double> newDoubleRange(String field, Double lowerVal, Double upperVal, boolean includeLower, boolean includeUpper) {
return new FieldCacheRangeFilter<Double>(field, lowerVal, upperVal, includeLower, includeUpper) {
public static DocValuesRangeFilter<Double> newDoubleRange(String field, Double lowerVal, Double upperVal, boolean includeLower, boolean includeUpper) {
return new DocValuesRangeFilter<Double>(field, lowerVal, upperVal, includeLower, includeUpper) {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
// we transform the floating point numbers to sortable integers
@ -363,7 +362,7 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
final NumericDocValues values = DocValues.getNumeric(context.reader(), field);
// ignore deleted docs if range doesn't contain 0
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected boolean matchDoc(int doc) {
final double value = Double.longBitsToDouble(values.get(doc));
@ -389,8 +388,8 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
@SuppressWarnings({"rawtypes"})
public final boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof FieldCacheRangeFilter)) return false;
FieldCacheRangeFilter other = (FieldCacheRangeFilter) o;
if (!(o instanceof DocValuesRangeFilter)) return false;
DocValuesRangeFilter other = (DocValuesRangeFilter) o;
if (!this.field.equals(other.field)
|| this.includeLower != other.includeLower

13
lucene/core/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java → lucene/core/src/java/org/apache/lucene/search/DocValuesRewriteMethod.java
View File

@ -34,24 +34,23 @@ import org.apache.lucene.util.LongBitSet;
* This can be used to perform these queries against an unindexed docvalues field.
* @lucene.experimental
*/
// nocommit: rename this class
public final class FieldCacheRewriteMethod extends MultiTermQuery.RewriteMethod {
public final class DocValuesRewriteMethod extends MultiTermQuery.RewriteMethod {
@Override
public Query rewrite(IndexReader reader, MultiTermQuery query) {
Query result = new ConstantScoreQuery(new MultiTermQueryFieldCacheWrapperFilter(query));
Query result = new ConstantScoreQuery(new MultiTermQueryDocValuesWrapperFilter(query));
result.setBoost(query.getBoost());
return result;
}
static class MultiTermQueryFieldCacheWrapperFilter extends Filter {
static class MultiTermQueryDocValuesWrapperFilter extends Filter {
protected final MultiTermQuery query;
/**
* Wrap a {@link MultiTermQuery} as a Filter.
*/
protected MultiTermQueryFieldCacheWrapperFilter(MultiTermQuery query) {
protected MultiTermQueryDocValuesWrapperFilter(MultiTermQuery query) {
this.query = query;
}
@ -66,7 +65,7 @@ public final class FieldCacheRewriteMethod extends MultiTermQuery.RewriteMethod
if (o==this) return true;
if (o==null) return false;
if (this.getClass().equals(o.getClass())) {
return this.query.equals( ((MultiTermQueryFieldCacheWrapperFilter)o).query );
return this.query.equals( ((MultiTermQueryDocValuesWrapperFilter)o).query );
}
return false;
}
@ -149,7 +148,7 @@ public final class FieldCacheRewriteMethod extends MultiTermQuery.RewriteMethod
return null;
}
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected final boolean matchDoc(int doc) throws ArrayIndexOutOfBoundsException {
int ord = fcsi.getOrd(doc);

8
lucene/core/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java → lucene/core/src/java/org/apache/lucene/search/DocValuesTermsFilter.java
View File

@ -87,16 +87,16 @@ import org.apache.lucene.util.FixedBitSet;
* Which filter is best is very application dependent.
*/
public class FieldCacheTermsFilter extends Filter {
public class DocValuesTermsFilter extends Filter {
private String field;
private BytesRef[] terms;
public FieldCacheTermsFilter(String field, BytesRef... terms) {
public DocValuesTermsFilter(String field, BytesRef... terms) {
this.field = field;
this.terms = terms;
}
public FieldCacheTermsFilter(String field, String... terms) {
public DocValuesTermsFilter(String field, String... terms) {
this.field = field;
this.terms = new BytesRef[terms.length];
for (int i = 0; i < terms.length; i++)
@ -113,7 +113,7 @@ public class FieldCacheTermsFilter extends Filter {
bits.set(ord);
}
}
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected final boolean matchDoc(int doc) {
int ord = fcsi.getOrd(doc);

4
lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java
View File

@ -84,7 +84,7 @@ public class FieldValueFilter extends Filter {
if (docsWithField instanceof MatchAllBits) {
return null;
}
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected final boolean matchDoc(int doc) {
return !docsWithField.get(doc);
@ -99,7 +99,7 @@ public class FieldValueFilter extends Filter {
// :-)
return BitsFilteredDocIdSet.wrap((DocIdSet) docsWithField, acceptDocs);
}
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected final boolean matchDoc(int doc) {
return docsWithField.get(doc);

2
lucene/core/src/java/org/apache/lucene/search/TermRangeFilter.java
View File

@ -29,7 +29,7 @@ import org.apache.lucene.util.BytesRef;
* for numerical ranges; use {@link NumericRangeFilter} instead.
*
* <p>If you construct a large number of range filters with different ranges but on the
* same field, {@link FieldCacheRangeFilter} may have significantly better performance.
* same field, {@link DocValuesRangeFilter} may have significantly better performance.
* @since 2.9
*/
public class TermRangeFilter extends MultiTermQueryWrapperFilter<TermRangeQuery> {

4
lucene/core/src/java/org/apache/lucene/search/package.html
View File

@ -337,8 +337,8 @@ extend by plugging in a different component (e.g. term frequency normalizer).
<p>
Finally, you can extend the low level {@link org.apache.lucene.search.similarities.Similarity Similarity} directly
to implement a new retrieval model, or to use external scoring factors particular to your application. For example,
a custom Similarity can access per-document values via {@link org.apache.lucene.search.FieldCache FieldCache} or
{@link org.apache.lucene.index.NumericDocValues} and integrate them into the score.
a custom Similarity can access per-document values via {@link org.apache.lucene.index.NumericDocValues} and
integrate them into the score.
</p>
<p>
See the {@link org.apache.lucene.search.similarities} package documentation for information

21
lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
View File

@ -63,6 +63,7 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.TestUtil;
import org.junit.AfterClass;
@ -818,22 +819,20 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
hits = searcher.search(NumericRangeQuery.newLongRange("trieLong", 4, Long.MIN_VALUE, Long.MAX_VALUE, false, false), 100).scoreDocs;
assertEquals("wrong number of hits", 34, hits.length);
// check decoding into field cache
// nocommit: instead use the NumericUtils termsenum stuff to test this directly...
/*
NumericDocValues fci = FieldCache.DEFAULT.getNumerics(SlowCompositeReaderWrapper.wrap(searcher.getIndexReader()), "trieInt", FieldCache.NUMERIC_UTILS_INT_PARSER, false);
int maxDoc = searcher.getIndexReader().maxDoc();
for(int doc=0;doc<maxDoc;doc++) {
long val = fci.get(doc);
// check decoding of terms
Terms terms = MultiFields.getTerms(searcher.getIndexReader(), "trieInt");
TermsEnum termsEnum = NumericUtils.filterPrefixCodedInts(terms.iterator(null));
while (termsEnum.next() != null) {
int val = NumericUtils.prefixCodedToInt(termsEnum.term());
assertTrue("value in id bounds", val >= 0 && val < 35);
}
NumericDocValues fcl = FieldCache.DEFAULT.getNumerics(SlowCompositeReaderWrapper.wrap(searcher.getIndexReader()), "trieLong", FieldCache.NUMERIC_UTILS_LONG_PARSER, false);
for(int doc=0;doc<maxDoc;doc++) {
long val = fcl.get(doc);
terms = MultiFields.getTerms(searcher.getIndexReader(), "trieLong");
termsEnum = NumericUtils.filterPrefixCodedLongs(terms.iterator(null));
while (termsEnum.next() != null) {
long val = NumericUtils.prefixCodedToLong(termsEnum.term());
assertTrue("value in id bounds", val >= 0L && val < 35L);
}
*/
reader.close();
}

2
lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
View File

@ -254,7 +254,7 @@ public class TestCachingWrapperFilter extends LuceneTestCase {
// returns default empty docidset, always cacheable:
assertDocIdSetCacheable(reader, NumericRangeFilter.newIntRange("test", Integer.valueOf(10000), Integer.valueOf(-10000), true, true), true);
// is cacheable:
assertDocIdSetCacheable(reader, FieldCacheRangeFilter.newIntRange("test", Integer.valueOf(10), Integer.valueOf(20), true, true), true);
assertDocIdSetCacheable(reader, DocValuesRangeFilter.newIntRange("test", Integer.valueOf(10), Integer.valueOf(20), true, true), true);
// a fixedbitset filter is always cacheable
assertDocIdSetCacheable(reader, new Filter() {
@Override

2
lucene/core/src/test/org/apache/lucene/search/TestExplanations.java
View File

@ -113,7 +113,7 @@ public class TestExplanations extends LuceneTestCase {
/**
* Convenience subclass of FieldCacheTermsFilter
*/
public static class ItemizedFilter extends FieldCacheTermsFilter {
public static class ItemizedFilter extends DocValuesTermsFilter {
private static String[] int2str(int [] terms) {
String [] out = new String[terms.length];
for (int i = 0; i < terms.length; i++) {

200
lucene/core/src/test/org/apache/lucene/search/TestFieldCacheRangeFilter.java
View File

@ -64,67 +64,67 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
Query q = new TermQuery(new Term("body","body"));
// test id, bounded on both ends
result = search.search(q, FieldCacheRangeFilter.newStringRange("id",minIP,maxIP,T,T), numDocs).scoreDocs;
result = search.search(q, DocValuesRangeFilter.newStringRange("id",minIP,maxIP,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",minIP,maxIP,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,maxIP,T,F), numDocs).scoreDocs;
assertEquals("all but last", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",minIP,maxIP,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,maxIP,F,T), numDocs).scoreDocs;
assertEquals("all but first", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",minIP,maxIP,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,maxIP,F,F), numDocs).scoreDocs;
assertEquals("all but ends", numDocs-2, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",medIP,maxIP,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",medIP,maxIP,T,T), numDocs).scoreDocs;
assertEquals("med and up", 1+ maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",minIP,medIP,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,medIP,T,T), numDocs).scoreDocs;
assertEquals("up to med", 1+ medId-minId, result.length);
// unbounded id
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",null,null,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",null,null,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",minIP,null,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,null,T,F), numDocs).scoreDocs;
assertEquals("min and up", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",null,maxIP,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",null,maxIP,F,T), numDocs).scoreDocs;
assertEquals("max and down", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",minIP,null,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,null,F,F), numDocs).scoreDocs;
assertEquals("not min, but up", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",null,maxIP,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",null,maxIP,F,F), numDocs).scoreDocs;
assertEquals("not max, but down", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",medIP,maxIP,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",medIP,maxIP,T,F), numDocs).scoreDocs;
assertEquals("med and up, not max", maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",minIP,medIP,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,medIP,F,T), numDocs).scoreDocs;
assertEquals("not min, up to med", medId-minId, result.length);
// very small sets
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",minIP,minIP,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,minIP,F,F), numDocs).scoreDocs;
assertEquals("min,min,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",medIP,medIP,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",medIP,medIP,F,F), numDocs).scoreDocs;
assertEquals("med,med,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",maxIP,maxIP,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",maxIP,maxIP,F,F), numDocs).scoreDocs;
assertEquals("max,max,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",minIP,minIP,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,minIP,T,T), numDocs).scoreDocs;
assertEquals("min,min,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",null,minIP,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",null,minIP,F,T), numDocs).scoreDocs;
assertEquals("nul,min,F,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",maxIP,maxIP,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",maxIP,maxIP,T,T), numDocs).scoreDocs;
assertEquals("max,max,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",maxIP,null,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",maxIP,null,T,F), numDocs).scoreDocs;
assertEquals("max,nul,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("id",medIP,medIP,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("id",medIP,medIP,T,T), numDocs).scoreDocs;
assertEquals("med,med,T,T", 1, result.length);
}
@ -146,47 +146,47 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
// test extremes, bounded on both ends
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",minRP,maxRP,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,maxRP,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",minRP,maxRP,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,maxRP,T,F), numDocs).scoreDocs;
assertEquals("all but biggest", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",minRP,maxRP,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,maxRP,F,T), numDocs).scoreDocs;
assertEquals("all but smallest", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",minRP,maxRP,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,maxRP,F,F), numDocs).scoreDocs;
assertEquals("all but extremes", numDocs-2, result.length);
// unbounded
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",minRP,null,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,null,T,F), numDocs).scoreDocs;
assertEquals("smallest and up", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",null,maxRP,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",null,maxRP,F,T), numDocs).scoreDocs;
assertEquals("biggest and down", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",minRP,null,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,null,F,F), numDocs).scoreDocs;
assertEquals("not smallest, but up", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",null,maxRP,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",null,maxRP,F,F), numDocs).scoreDocs;
assertEquals("not biggest, but down", numDocs-1, result.length);
// very small sets
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",minRP,minRP,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,minRP,F,F), numDocs).scoreDocs;
assertEquals("min,min,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",maxRP,maxRP,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",maxRP,maxRP,F,F), numDocs).scoreDocs;
assertEquals("max,max,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",minRP,minRP,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,minRP,T,T), numDocs).scoreDocs;
assertEquals("min,min,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",null,minRP,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",null,minRP,F,T), numDocs).scoreDocs;
assertEquals("nul,min,F,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",maxRP,maxRP,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",maxRP,maxRP,T,T), numDocs).scoreDocs;
assertEquals("max,max,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newStringRange("rand",maxRP,null,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newStringRange("rand",maxRP,null,T,F), numDocs).scoreDocs;
assertEquals("max,nul,T,T", 1, result.length);
}
@ -209,75 +209,75 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
// test id, bounded on both ends
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,maxIdO,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,maxIdO,T,F), numDocs).scoreDocs;
assertEquals("all but last", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,maxIdO,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,maxIdO,F,T), numDocs).scoreDocs;
assertEquals("all but first", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("all but ends", numDocs-2, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",medIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",medIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("med and up", 1+ maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("up to med", 1+ medId-minId, result.length);
// unbounded id
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",null,null,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",null,null,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,null,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,null,T,F), numDocs).scoreDocs;
assertEquals("min and up", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",null,maxIdO,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",null,maxIdO,F,T), numDocs).scoreDocs;
assertEquals("max and down", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,null,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,null,F,F), numDocs).scoreDocs;
assertEquals("not min, but up", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",null,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",null,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("not max, but down", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",medIdO,maxIdO,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",medIdO,maxIdO,T,F), numDocs).scoreDocs;
assertEquals("med and up, not max", maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,medIdO,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,medIdO,F,T), numDocs).scoreDocs;
assertEquals("not min, up to med", medId-minId, result.length);
// very small sets
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,minIdO,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,minIdO,F,F), numDocs).scoreDocs;
assertEquals("min,min,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",medIdO,medIdO,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",medIdO,medIdO,F,F), numDocs).scoreDocs;
assertEquals("med,med,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",maxIdO,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",maxIdO,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("max,max,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,minIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,minIdO,T,T), numDocs).scoreDocs;
assertEquals("min,min,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",null,minIdO,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",null,minIdO,F,T), numDocs).scoreDocs;
assertEquals("nul,min,F,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",maxIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",maxIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("max,max,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",maxIdO,null,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",maxIdO,null,T,F), numDocs).scoreDocs;
assertEquals("max,nul,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",medIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",medIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("med,med,T,T", 1, result.length);
// special cases
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",Integer.valueOf(Integer.MAX_VALUE),null,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",Integer.valueOf(Integer.MAX_VALUE),null,F,F), numDocs).scoreDocs;
assertEquals("overflow special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",null,Integer.valueOf(Integer.MIN_VALUE),F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",null,Integer.valueOf(Integer.MIN_VALUE),F,F), numDocs).scoreDocs;
assertEquals("overflow special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",maxIdO,minIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",maxIdO,minIdO,T,T), numDocs).scoreDocs;
assertEquals("inverse range", 0, result.length);
}
@ -300,75 +300,75 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
// test id, bounded on both ends
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,maxIdO,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,maxIdO,T,F), numDocs).scoreDocs;
assertEquals("all but last", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,maxIdO,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,maxIdO,F,T), numDocs).scoreDocs;
assertEquals("all but first", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("all but ends", numDocs-2, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",medIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",medIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("med and up", 1+ maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("up to med", 1+ medId-minId, result.length);
// unbounded id
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",null,null,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",null,null,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,null,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,null,T,F), numDocs).scoreDocs;
assertEquals("min and up", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",null,maxIdO,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",null,maxIdO,F,T), numDocs).scoreDocs;
assertEquals("max and down", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,null,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,null,F,F), numDocs).scoreDocs;
assertEquals("not min, but up", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",null,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",null,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("not max, but down", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",medIdO,maxIdO,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",medIdO,maxIdO,T,F), numDocs).scoreDocs;
assertEquals("med and up, not max", maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,medIdO,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,medIdO,F,T), numDocs).scoreDocs;
assertEquals("not min, up to med", medId-minId, result.length);
// very small sets
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,minIdO,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,minIdO,F,F), numDocs).scoreDocs;
assertEquals("min,min,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",medIdO,medIdO,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",medIdO,medIdO,F,F), numDocs).scoreDocs;
assertEquals("med,med,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",maxIdO,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",maxIdO,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("max,max,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,minIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,minIdO,T,T), numDocs).scoreDocs;
assertEquals("min,min,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",null,minIdO,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",null,minIdO,F,T), numDocs).scoreDocs;
assertEquals("nul,min,F,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",maxIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",maxIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("max,max,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",maxIdO,null,T,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",maxIdO,null,T,F), numDocs).scoreDocs;
assertEquals("max,nul,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",medIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",medIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("med,med,T,T", 1, result.length);
// special cases
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",Long.valueOf(Long.MAX_VALUE),null,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",Long.valueOf(Long.MAX_VALUE),null,F,F), numDocs).scoreDocs;
assertEquals("overflow special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",null,Long.valueOf(Long.MIN_VALUE),F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",null,Long.valueOf(Long.MIN_VALUE),F,F), numDocs).scoreDocs;
assertEquals("overflow special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",maxIdO,minIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",maxIdO,minIdO,T,T), numDocs).scoreDocs;
assertEquals("inverse range", 0, result.length);
}
@ -387,19 +387,19 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
ScoreDoc[] result;
Query q = new TermQuery(new Term("body","body"));
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",minIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",minIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs/2, result.length);
int count = 0;
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",null,medIdO,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",null,medIdO,F,T), numDocs).scoreDocs;
count += result.length;
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",medIdO,null,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",medIdO,null,F,F), numDocs).scoreDocs;
count += result.length;
assertEquals("sum of two concenatted ranges", numDocs, count);
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",null,null,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",null,null,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",Float.valueOf(Float.POSITIVE_INFINITY),null,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",Float.valueOf(Float.POSITIVE_INFINITY),null,F,F), numDocs).scoreDocs;
assertEquals("infinity special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",null,Float.valueOf(Float.NEGATIVE_INFINITY),F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",null,Float.valueOf(Float.NEGATIVE_INFINITY),F,F), numDocs).scoreDocs;
assertEquals("infinity special case", 0, result.length);
}
@ -416,19 +416,19 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
ScoreDoc[] result;
Query q = new TermQuery(new Term("body","body"));
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",minIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",minIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs/2, result.length);
int count = 0;
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",null,medIdO,F,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",null,medIdO,F,T), numDocs).scoreDocs;
count += result.length;
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",medIdO,null,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",medIdO,null,F,F), numDocs).scoreDocs;
count += result.length;
assertEquals("sum of two concenatted ranges", numDocs, count);
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",null,null,T,T), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",null,null,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",Double.valueOf(Double.POSITIVE_INFINITY),null,F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",Double.valueOf(Double.POSITIVE_INFINITY),null,F,F), numDocs).scoreDocs;
assertEquals("infinity special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",null, Double.valueOf(Double.NEGATIVE_INFINITY),F,F), numDocs).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",null, Double.valueOf(Double.NEGATIVE_INFINITY),F,F), numDocs).scoreDocs;
assertEquals("infinity special case", 0, result.length);
}
@ -459,19 +459,19 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
ScoreDoc[] result;
Query q = new TermQuery(new Term("body","body"));
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",-20,20,T,T), 100).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",-20,20,T,T), 100).scoreDocs;
assertEquals("find all", 40, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",0,20,T,T), 100).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",0,20,T,T), 100).scoreDocs;
assertEquals("find all", 20, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",-20,0,T,T), 100).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",-20,0,T,T), 100).scoreDocs;
assertEquals("find all", 20, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",10,20,T,T), 100).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",10,20,T,T), 100).scoreDocs;
assertEquals("find all", 11, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",-20,-10,T,T), 100).scoreDocs;
result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",-20,-10,T,T), 100).scoreDocs;
assertEquals("find all", 11, result.length);
reader.close();
dir.close();

8
lucene/core/src/test/org/apache/lucene/search/TestFieldCacheRewriteMethod.java
View File

@ -31,7 +31,7 @@ public class TestFieldCacheRewriteMethod extends TestRegexpRandom2 {
@Override
protected void assertSame(String regexp) throws IOException {
RegexpQuery fieldCache = new RegexpQuery(new Term(fieldName, regexp), RegExp.NONE);
fieldCache.setRewriteMethod(new FieldCacheRewriteMethod());
fieldCache.setRewriteMethod(new DocValuesRewriteMethod());
RegexpQuery filter = new RegexpQuery(new Term(fieldName, regexp), RegExp.NONE);
filter.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
@ -49,9 +49,9 @@ public class TestFieldCacheRewriteMethod extends TestRegexpRandom2 {
assertEquals(a1, a2);
assertFalse(a1.equals(b));
a1.setRewriteMethod(new FieldCacheRewriteMethod());
a2.setRewriteMethod(new FieldCacheRewriteMethod());
b.setRewriteMethod(new FieldCacheRewriteMethod());
a1.setRewriteMethod(new DocValuesRewriteMethod());
a2.setRewriteMethod(new DocValuesRewriteMethod());
b.setRewriteMethod(new DocValuesRewriteMethod());
assertEquals(a1, a2);
assertFalse(a1.equals(b));
QueryUtils.check(a1);

8
lucene/core/src/test/org/apache/lucene/search/TestFieldCacheTermsFilter.java
View File

@ -33,7 +33,7 @@ import java.util.List;
/**
* A basic unit test for FieldCacheTermsFilter
*
* @see org.apache.lucene.search.FieldCacheTermsFilter
* @see org.apache.lucene.search.DocValuesTermsFilter
*/
public class TestFieldCacheTermsFilter extends LuceneTestCase {
public void testMissingTerms() throws Exception {
@ -58,18 +58,18 @@ public class TestFieldCacheTermsFilter extends LuceneTestCase {
List<String> terms = new ArrayList<>();
terms.add("5");
results = searcher.search(q, new FieldCacheTermsFilter(fieldName, terms.toArray(new String[0])), numDocs).scoreDocs;
results = searcher.search(q, new DocValuesTermsFilter(fieldName, terms.toArray(new String[0])), numDocs).scoreDocs;
assertEquals("Must match nothing", 0, results.length);
terms = new ArrayList<>();
terms.add("10");
results = searcher.search(q, new FieldCacheTermsFilter(fieldName, terms.toArray(new String[0])), numDocs).scoreDocs;
results = searcher.search(q, new DocValuesTermsFilter(fieldName, terms.toArray(new String[0])), numDocs).scoreDocs;
assertEquals("Must match 1", 1, results.length);
terms = new ArrayList<>();
terms.add("10");
terms.add("20");
results = searcher.search(q, new FieldCacheTermsFilter(fieldName, terms.toArray(new String[0])), numDocs).scoreDocs;
results = searcher.search(q, new DocValuesTermsFilter(fieldName, terms.toArray(new String[0])), numDocs).scoreDocs;
assertEquals("Must match 2", 2, results.length);
reader.close();

83
lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailOnFieldCacheInsanity.java
View File

@ -1,83 +0,0 @@
package org.apache.lucene.util.junitcompat;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.JUnitCore;
import org.junit.runner.Result;
import org.junit.runner.notification.Failure;
public class TestFailOnFieldCacheInsanity extends WithNestedTests {
public TestFailOnFieldCacheInsanity() {
super(true);
}
public static class Nested1 extends WithNestedTests.AbstractNestedTest {
private Directory d;
private IndexReader r;
private AtomicReader subR;
private void makeIndex() throws Exception {
// we use RAMDirectory here, because we dont want to stay on open files on Windows:
d = new RAMDirectory();
@SuppressWarnings("resource") RandomIndexWriter w =
new RandomIndexWriter(random(), d);
Document doc = new Document();
doc.add(newField("ints", "1", StringField.TYPE_NOT_STORED));
w.addDocument(doc);
w.forceMerge(1);
r = w.getReader();
w.shutdown();
subR = r.leaves().get(0).reader();
}
public void testDummy() throws Exception {
makeIndex();
/* nocommit
assertNotNull(FieldCache.DEFAULT.getTermsIndex(subR, "ints"));
assertNotNull(FieldCache.DEFAULT.getTerms(subR, "ints", false));
*/
// NOTE: do not close reader/directory, else it
// purges FC entries
}
}
// nocommit: move this to solr?
@Test @Ignore
public void testFailOnFieldCacheInsanity() {
Result r = JUnitCore.runClasses(Nested1.class);
boolean insane = false;
for(Failure f : r.getFailures()) {
if (f.getMessage().indexOf("Insane") != -1) {
insane = true;
break;
}
}
Assert.assertTrue(insane);
}
}

146
lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
View File

@ -299,83 +299,79 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
fieldIdToDocID[fieldId] = i;
}
try {
final IndexSearcher s = newSearcher(r);
for (int contentID = 0; contentID < 3; contentID++) {
final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real" + contentID)), numDocs).scoreDocs;
for (ScoreDoc hit : hits) {
final GroupDoc gd = groupDocs[(int) docIdToFieldId.get(hit.doc)];
assertTrue(gd.score == 0.0);
gd.score = hit.score;
int docId = gd.id;
assertEquals(docId, docIdToFieldId.get(hit.doc));
}
final IndexSearcher s = newSearcher(r);
for (int contentID = 0; contentID < 3; contentID++) {
final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real" + contentID)), numDocs).scoreDocs;
for (ScoreDoc hit : hits) {
final GroupDoc gd = groupDocs[(int) docIdToFieldId.get(hit.doc)];
assertTrue(gd.score == 0.0);
gd.score = hit.score;
int docId = gd.id;
assertEquals(docId, docIdToFieldId.get(hit.doc));
}
for (GroupDoc gd : groupDocs) {
assertTrue(gd.score != 0.0);
}
for (int searchIter = 0; searchIter < 100; searchIter++) {
if (VERBOSE) {
System.out.println("TEST: searchIter=" + searchIter);
}
final String searchTerm = "real" + random().nextInt(3);
boolean sortByScoreOnly = random().nextBoolean();
Sort sortWithinGroup = getRandomSort(sortByScoreOnly);
AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = createRandomCollector("group", sortWithinGroup);
s.search(new TermQuery(new Term("content", searchTerm)), allGroupHeadsCollector);
int[] expectedGroupHeads = createExpectedGroupHeads(searchTerm, groupDocs, sortWithinGroup, sortByScoreOnly, fieldIdToDocID);
int[] actualGroupHeads = allGroupHeadsCollector.retrieveGroupHeads();
// The actual group heads contains Lucene ids. Need to change them into our id value.
for (int i = 0; i < actualGroupHeads.length; i++) {
actualGroupHeads[i] = (int) docIdToFieldId.get(actualGroupHeads[i]);
}
// Allows us the easily iterate and assert the actual and expected results.
Arrays.sort(expectedGroupHeads);
Arrays.sort(actualGroupHeads);
if (VERBOSE) {
System.out.println("Collector: " + allGroupHeadsCollector.getClass().getSimpleName());
System.out.println("Sort within group: " + sortWithinGroup);
System.out.println("Num group: " + numGroups);
System.out.println("Num doc: " + numDocs);
System.out.println("\n=== Expected: \n");
for (int expectedDocId : expectedGroupHeads) {
GroupDoc expectedGroupDoc = groupDocs[expectedDocId];
String expectedGroup = expectedGroupDoc.group == null ? null : expectedGroupDoc.group.utf8ToString();
System.out.println(
String.format(Locale.ROOT,
"Group:%10s score%5f Sort1:%10s Sort2:%10s Sort3:%10s doc:%5d",
expectedGroup, expectedGroupDoc.score, expectedGroupDoc.sort1.utf8ToString(),
expectedGroupDoc.sort2.utf8ToString(), expectedGroupDoc.sort3.utf8ToString(), expectedDocId
)
);
}
System.out.println("\n=== Actual: \n");
for (int actualDocId : actualGroupHeads) {
GroupDoc actualGroupDoc = groupDocs[actualDocId];
String actualGroup = actualGroupDoc.group == null ? null : actualGroupDoc.group.utf8ToString();
System.out.println(
String.format(Locale.ROOT,
"Group:%10s score%5f Sort1:%10s Sort2:%10s Sort3:%10s doc:%5d",
actualGroup, actualGroupDoc.score, actualGroupDoc.sort1.utf8ToString(),
actualGroupDoc.sort2.utf8ToString(), actualGroupDoc.sort3.utf8ToString(), actualDocId
)
);
}
System.out.println("\n===================================================================================");
}
assertArrayEquals(expectedGroupHeads, actualGroupHeads);
}
} finally {
QueryUtils.purgeFieldCache(r);
}
for (GroupDoc gd : groupDocs) {
assertTrue(gd.score != 0.0);
}
for (int searchIter = 0; searchIter < 100; searchIter++) {
if (VERBOSE) {
System.out.println("TEST: searchIter=" + searchIter);
}
final String searchTerm = "real" + random().nextInt(3);
boolean sortByScoreOnly = random().nextBoolean();
Sort sortWithinGroup = getRandomSort(sortByScoreOnly);
AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = createRandomCollector("group", sortWithinGroup);
s.search(new TermQuery(new Term("content", searchTerm)), allGroupHeadsCollector);
int[] expectedGroupHeads = createExpectedGroupHeads(searchTerm, groupDocs, sortWithinGroup, sortByScoreOnly, fieldIdToDocID);
int[] actualGroupHeads = allGroupHeadsCollector.retrieveGroupHeads();
// The actual group heads contains Lucene ids. Need to change them into our id value.
for (int i = 0; i < actualGroupHeads.length; i++) {
actualGroupHeads[i] = (int) docIdToFieldId.get(actualGroupHeads[i]);
}
// Allows us the easily iterate and assert the actual and expected results.
Arrays.sort(expectedGroupHeads);
Arrays.sort(actualGroupHeads);
if (VERBOSE) {
System.out.println("Collector: " + allGroupHeadsCollector.getClass().getSimpleName());
System.out.println("Sort within group: " + sortWithinGroup);
System.out.println("Num group: " + numGroups);
System.out.println("Num doc: " + numDocs);
System.out.println("\n=== Expected: \n");
for (int expectedDocId : expectedGroupHeads) {
GroupDoc expectedGroupDoc = groupDocs[expectedDocId];
String expectedGroup = expectedGroupDoc.group == null ? null : expectedGroupDoc.group.utf8ToString();
System.out.println(
String.format(Locale.ROOT,
"Group:%10s score%5f Sort1:%10s Sort2:%10s Sort3:%10s doc:%5d",
expectedGroup, expectedGroupDoc.score, expectedGroupDoc.sort1.utf8ToString(),
expectedGroupDoc.sort2.utf8ToString(), expectedGroupDoc.sort3.utf8ToString(), expectedDocId
)
);
}
System.out.println("\n=== Actual: \n");
for (int actualDocId : actualGroupHeads) {
GroupDoc actualGroupDoc = groupDocs[actualDocId];
String actualGroup = actualGroupDoc.group == null ? null : actualGroupDoc.group.utf8ToString();
System.out.println(
String.format(Locale.ROOT,
"Group:%10s score%5f Sort1:%10s Sort2:%10s Sort3:%10s doc:%5d",
actualGroup, actualGroupDoc.score, actualGroupDoc.sort1.utf8ToString(),
actualGroupDoc.sort2.utf8ToString(), actualGroupDoc.sort3.utf8ToString(), actualDocId
)
);
}
System.out.println("\n===================================================================================");
}
assertArrayEquals(expectedGroupHeads, actualGroupHeads);
}
r.close();
dir.close();
}

753
lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
View File

@ -748,390 +748,383 @@ public class TestGrouping extends LuceneTestCase {
DirectoryReader rBlocks = null;
Directory dirBlocks = null;
try {
final IndexSearcher s = newSearcher(r);
if (VERBOSE) {
System.out.println("\nTEST: searcher=" + s);
}
final ShardState shards = new ShardState(s);
for(int contentID=0;contentID<3;contentID++) {
final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
for(ScoreDoc hit : hits) {
final GroupDoc gd = groupDocs[(int) docIDToID.get(hit.doc)];
assertTrue(gd.score == 0.0);
gd.score = hit.score;
assertEquals(gd.id, docIDToID.get(hit.doc));
}
}
for(GroupDoc gd : groupDocs) {
assertTrue(gd.score != 0.0);
}
// Build 2nd index, where docs are added in blocks by
// group, so we can use single pass collector
dirBlocks = newDirectory();
rBlocks = getDocBlockReader(dirBlocks, groupDocs);
final Filter lastDocInBlock = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupend", "x"))));
final NumericDocValues docIDToIDBlocks = MultiDocValues.getNumericValues(rBlocks, "id");
assertNotNull(docIDToIDBlocks);
final IndexSearcher sBlocks = newSearcher(rBlocks);
final ShardState shardsBlocks = new ShardState(sBlocks);
// ReaderBlocks only increases maxDoc() vs reader, which
// means a monotonic shift in scores, so we can
// reliably remap them w/ Map:
final Map<String,Map<Float,Float>> scoreMap = new HashMap<>();
// Tricky: must separately set .score2, because the doc
// block index was created with possible deletions!
//System.out.println("fixup score2");
for(int contentID=0;contentID<3;contentID++) {
//System.out.println(" term=real" + contentID);
final Map<Float,Float> termScoreMap = new HashMap<>();
scoreMap.put("real"+contentID, termScoreMap);
//System.out.println("term=real" + contentID + " dfold=" + s.docFreq(new Term("content", "real"+contentID)) +
//" dfnew=" + sBlocks.docFreq(new Term("content", "real"+contentID)));
final ScoreDoc[] hits = sBlocks.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
for(ScoreDoc hit : hits) {
final GroupDoc gd = groupDocsByID[(int) docIDToIDBlocks.get(hit.doc)];
assertTrue(gd.score2 == 0.0);
gd.score2 = hit.score;
assertEquals(gd.id, docIDToIDBlocks.get(hit.doc));
//System.out.println(" score=" + gd.score + " score2=" + hit.score + " id=" + docIDToIDBlocks.get(hit.doc));
termScoreMap.put(gd.score, gd.score2);
}
}
for(int searchIter=0;searchIter<100;searchIter++) {
if (VERBOSE) {
System.out.println("\nTEST: searchIter=" + searchIter);
}
final String searchTerm = "real" + random().nextInt(3);
final boolean fillFields = random().nextBoolean();
boolean getScores = random().nextBoolean();
final boolean getMaxScores = random().nextBoolean();
final Sort groupSort = getRandomSort();
//final Sort groupSort = new Sort(new SortField[] {new SortField("sort1", SortField.STRING), new SortField("id", SortField.INT)});
// TODO: also test null (= sort by relevance)
final Sort docSort = getRandomSort();
for(SortField sf : docSort.getSort()) {
if (sf.getType() == SortField.Type.SCORE) {
getScores = true;
break;
}
}
for(SortField sf : groupSort.getSort()) {
if (sf.getType() == SortField.Type.SCORE) {
getScores = true;
break;
}
}
final int topNGroups = TestUtil.nextInt(random(), 1, 30);
//final int topNGroups = 10;
final int docsPerGroup = TestUtil.nextInt(random(), 1, 50);
final int groupOffset = TestUtil.nextInt(random(), 0, (topNGroups - 1) / 2);
//final int groupOffset = 0;
final int docOffset = TestUtil.nextInt(random(), 0, docsPerGroup - 1);
//final int docOffset = 0;
final boolean doCache = random().nextBoolean();
final boolean doAllGroups = random().nextBoolean();
if (VERBOSE) {
System.out.println("TEST: groupSort=" + groupSort + " docSort=" + docSort + " searchTerm=" + searchTerm + " dF=" + r.docFreq(new Term("content", searchTerm)) +" dFBlock=" + rBlocks.docFreq(new Term("content", searchTerm)) + " topNGroups=" + topNGroups + " groupOffset=" + groupOffset + " docOffset=" + docOffset + " doCache=" + doCache + " docsPerGroup=" + docsPerGroup + " doAllGroups=" + doAllGroups + " getScores=" + getScores + " getMaxScores=" + getMaxScores);
}
String groupField = "group";
if (VERBOSE) {
System.out.println(" groupField=" + groupField);
}
final AbstractFirstPassGroupingCollector<?> c1 = createRandomFirstPassCollector(groupField, groupSort, groupOffset+topNGroups);
final CachingCollector cCache;
final Collector c;
final AbstractAllGroupsCollector<?> allGroupsCollector;
if (doAllGroups) {
allGroupsCollector = createAllGroupsCollector(c1, groupField);
} else {
allGroupsCollector = null;
}
final boolean useWrappingCollector = random().nextBoolean();
if (doCache) {
final double maxCacheMB = random().nextDouble();
if (VERBOSE) {
System.out.println("TEST: maxCacheMB=" + maxCacheMB);
}
if (useWrappingCollector) {
if (doAllGroups) {
cCache = CachingCollector.create(c1, true, maxCacheMB);
c = MultiCollector.wrap(cCache, allGroupsCollector);
} else {
c = cCache = CachingCollector.create(c1, true, maxCacheMB);
}
} else {
// Collect only into cache, then replay multiple times:
c = cCache = CachingCollector.create(false, true, maxCacheMB);
}
} else {
cCache = null;
if (doAllGroups) {
c = MultiCollector.wrap(c1, allGroupsCollector);
} else {
c = c1;
}
}
// Search top reader:
final Query query = new TermQuery(new Term("content", searchTerm));
s.search(query, c);
if (doCache && !useWrappingCollector) {
if (cCache.isCached()) {
// Replay for first-pass grouping
cCache.replay(c1);
if (doAllGroups) {
// Replay for all groups:
cCache.replay(allGroupsCollector);
}
} else {
// Replay by re-running search:
s.search(query, c1);
if (doAllGroups) {
s.search(query, allGroupsCollector);
}
}
}
// Get 1st pass top groups
final Collection<SearchGroup<BytesRef>> topGroups = getSearchGroups(c1, groupOffset, fillFields);
final TopGroups<BytesRef> groupsResult;
if (VERBOSE) {
System.out.println("TEST: first pass topGroups");
if (topGroups == null) {
System.out.println(" null");
} else {
for (SearchGroup<BytesRef> searchGroup : topGroups) {
System.out.println(" " + (searchGroup.groupValue == null ? "null" : searchGroup.groupValue) + ": " + Arrays.deepToString(searchGroup.sortValues));
}
}
}
// Get 1st pass top groups using shards
final TopGroups<BytesRef> topGroupsShards = searchShards(s, shards.subSearchers, query, groupSort, docSort,
groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, true, false);
final AbstractSecondPassGroupingCollector<?> c2;
if (topGroups != null) {
if (VERBOSE) {
System.out.println("TEST: topGroups");
for (SearchGroup<BytesRef> searchGroup : topGroups) {
System.out.println(" " + (searchGroup.groupValue == null ? "null" : searchGroup.groupValue.utf8ToString()) + ": " + Arrays.deepToString(searchGroup.sortValues));
}
}
c2 = createSecondPassCollector(c1, groupField, groupSort, docSort, groupOffset, docOffset + docsPerGroup, getScores, getMaxScores, fillFields);
if (doCache) {
if (cCache.isCached()) {
if (VERBOSE) {
System.out.println("TEST: cache is intact");
}
cCache.replay(c2);
} else {
if (VERBOSE) {
System.out.println("TEST: cache was too large");
}
s.search(query, c2);
}
} else {
s.search(query, c2);
}
if (doAllGroups) {
TopGroups<BytesRef> tempTopGroups = getTopGroups(c2, docOffset);
groupsResult = new TopGroups<>(tempTopGroups, allGroupsCollector.getGroupCount());
} else {
groupsResult = getTopGroups(c2, docOffset);
}
} else {
c2 = null;
groupsResult = null;
if (VERBOSE) {
System.out.println("TEST: no results");
}
}
final TopGroups<BytesRef> expectedGroups = slowGrouping(groupDocs, searchTerm, fillFields, getScores, getMaxScores, doAllGroups, groupSort, docSort, topNGroups, docsPerGroup, groupOffset, docOffset);
if (VERBOSE) {
if (expectedGroups == null) {
System.out.println("TEST: no expected groups");
} else {
System.out.println("TEST: expected groups totalGroupedHitCount=" + expectedGroups.totalGroupedHitCount);
for(GroupDocs<BytesRef> gd : expectedGroups.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits + " scoreDocs.len=" + gd.scoreDocs.length);
for(ScoreDoc sd : gd.scoreDocs) {
System.out.println(" id=" + sd.doc + " score=" + sd.score);
}
}
}
if (groupsResult == null) {
System.out.println("TEST: no matched groups");
} else {
System.out.println("TEST: matched groups totalGroupedHitCount=" + groupsResult.totalGroupedHitCount);
for(GroupDocs<BytesRef> gd : groupsResult.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
}
}
if (searchIter == 14) {
for(int docIDX=0;docIDX<s.getIndexReader().maxDoc();docIDX++) {
System.out.println("ID=" + docIDToID.get(docIDX) + " explain=" + s.explain(query, docIDX));
}
}
}
if (topGroupsShards == null) {
System.out.println("TEST: no matched-merged groups");
} else {
System.out.println("TEST: matched-merged groups totalGroupedHitCount=" + topGroupsShards.totalGroupedHitCount);
for(GroupDocs<BytesRef> gd : topGroupsShards.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
}
}
}
}
assertEquals(docIDToID, expectedGroups, groupsResult, true, true, true, getScores, true);
// Confirm merged shards match:
assertEquals(docIDToID, expectedGroups, topGroupsShards, true, false, fillFields, getScores, true);
if (topGroupsShards != null) {
verifyShards(shards.docStarts, topGroupsShards);
}
final boolean needsScores = getScores || getMaxScores || docSort == null;
final BlockGroupingCollector c3 = new BlockGroupingCollector(groupSort, groupOffset+topNGroups, needsScores, lastDocInBlock);
final TermAllGroupsCollector allGroupsCollector2;
final Collector c4;
if (doAllGroups) {
// NOTE: must be "group" and not "group_dv"
// (groupField) because we didn't index doc
// values in the block index:
allGroupsCollector2 = new TermAllGroupsCollector("group");
c4 = MultiCollector.wrap(c3, allGroupsCollector2);
} else {
allGroupsCollector2 = null;
c4 = c3;
}
// Get block grouping result:
sBlocks.search(query, c4);
@SuppressWarnings({"unchecked","rawtypes"})
final TopGroups<BytesRef> tempTopGroupsBlocks = (TopGroups<BytesRef>) c3.getTopGroups(docSort, groupOffset, docOffset, docOffset+docsPerGroup, fillFields);
final TopGroups<BytesRef> groupsResultBlocks;
if (doAllGroups && tempTopGroupsBlocks != null) {
assertEquals((int) tempTopGroupsBlocks.totalGroupCount, allGroupsCollector2.getGroupCount());
groupsResultBlocks = new TopGroups<>(tempTopGroupsBlocks, allGroupsCollector2.getGroupCount());
} else {
groupsResultBlocks = tempTopGroupsBlocks;
}
if (VERBOSE) {
if (groupsResultBlocks == null) {
System.out.println("TEST: no block groups");
} else {
System.out.println("TEST: block groups totalGroupedHitCount=" + groupsResultBlocks.totalGroupedHitCount);
boolean first = true;
for(GroupDocs<BytesRef> gd : groupsResultBlocks.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue.utf8ToString()) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
System.out.println(" id=" + docIDToIDBlocks.get(sd.doc) + " score=" + sd.score);
if (first) {
System.out.println("explain: " + sBlocks.explain(query, sd.doc));
first = false;
}
}
}
}
}
// Get shard'd block grouping result:
final TopGroups<BytesRef> topGroupsBlockShards = searchShards(sBlocks, shardsBlocks.subSearchers, query,
groupSort, docSort, groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, false, false);
if (expectedGroups != null) {
// Fixup scores for reader2
for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
for(ScoreDoc hit : groupDocsHits.scoreDocs) {
final GroupDoc gd = groupDocsByID[hit.doc];
assertEquals(gd.id, hit.doc);
//System.out.println("fixup score " + hit.score + " to " + gd.score2 + " vs " + gd.score);
hit.score = gd.score2;
}
}
final SortField[] sortFields = groupSort.getSort();
final Map<Float,Float> termScoreMap = scoreMap.get(searchTerm);
for(int groupSortIDX=0;groupSortIDX<sortFields.length;groupSortIDX++) {
if (sortFields[groupSortIDX].getType() == SortField.Type.SCORE) {
for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
if (groupDocsHits.groupSortValues != null) {
//System.out.println("remap " + groupDocsHits.groupSortValues[groupSortIDX] + " to " + termScoreMap.get(groupDocsHits.groupSortValues[groupSortIDX]));
groupDocsHits.groupSortValues[groupSortIDX] = termScoreMap.get(groupDocsHits.groupSortValues[groupSortIDX]);
assertNotNull(groupDocsHits.groupSortValues[groupSortIDX]);
}
}
}
}
final SortField[] docSortFields = docSort.getSort();
for(int docSortIDX=0;docSortIDX<docSortFields.length;docSortIDX++) {
if (docSortFields[docSortIDX].getType() == SortField.Type.SCORE) {
for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
for(ScoreDoc _hit : groupDocsHits.scoreDocs) {
FieldDoc hit = (FieldDoc) _hit;
if (hit.fields != null) {
hit.fields[docSortIDX] = termScoreMap.get(hit.fields[docSortIDX]);
assertNotNull(hit.fields[docSortIDX]);
}
}
}
}
}
}
assertEquals(docIDToIDBlocks, expectedGroups, groupsResultBlocks, false, true, true, getScores, false);
assertEquals(docIDToIDBlocks, expectedGroups, topGroupsBlockShards, false, false, fillFields, getScores, false);
}
} finally {
QueryUtils.purgeFieldCache(r);
if (rBlocks != null) {
QueryUtils.purgeFieldCache(rBlocks);
final IndexSearcher s = newSearcher(r);
if (VERBOSE) {
System.out.println("\nTEST: searcher=" + s);
}
final ShardState shards = new ShardState(s);
for(int contentID=0;contentID<3;contentID++) {
final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
for(ScoreDoc hit : hits) {
final GroupDoc gd = groupDocs[(int) docIDToID.get(hit.doc)];
assertTrue(gd.score == 0.0);
gd.score = hit.score;
assertEquals(gd.id, docIDToID.get(hit.doc));
}
}
for(GroupDoc gd : groupDocs) {
assertTrue(gd.score != 0.0);
}
// Build 2nd index, where docs are added in blocks by
// group, so we can use single pass collector
dirBlocks = newDirectory();
rBlocks = getDocBlockReader(dirBlocks, groupDocs);
final Filter lastDocInBlock = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupend", "x"))));
final NumericDocValues docIDToIDBlocks = MultiDocValues.getNumericValues(rBlocks, "id");
assertNotNull(docIDToIDBlocks);
final IndexSearcher sBlocks = newSearcher(rBlocks);
final ShardState shardsBlocks = new ShardState(sBlocks);
// ReaderBlocks only increases maxDoc() vs reader, which
// means a monotonic shift in scores, so we can
// reliably remap them w/ Map:
final Map<String,Map<Float,Float>> scoreMap = new HashMap<>();
// Tricky: must separately set .score2, because the doc
// block index was created with possible deletions!
//System.out.println("fixup score2");
for(int contentID=0;contentID<3;contentID++) {
//System.out.println(" term=real" + contentID);
final Map<Float,Float> termScoreMap = new HashMap<>();
scoreMap.put("real"+contentID, termScoreMap);
//System.out.println("term=real" + contentID + " dfold=" + s.docFreq(new Term("content", "real"+contentID)) +
//" dfnew=" + sBlocks.docFreq(new Term("content", "real"+contentID)));
final ScoreDoc[] hits = sBlocks.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
for(ScoreDoc hit : hits) {
final GroupDoc gd = groupDocsByID[(int) docIDToIDBlocks.get(hit.doc)];
assertTrue(gd.score2 == 0.0);
gd.score2 = hit.score;
assertEquals(gd.id, docIDToIDBlocks.get(hit.doc));
//System.out.println(" score=" + gd.score + " score2=" + hit.score + " id=" + docIDToIDBlocks.get(hit.doc));
termScoreMap.put(gd.score, gd.score2);
}
}
for(int searchIter=0;searchIter<100;searchIter++) {
if (VERBOSE) {
System.out.println("\nTEST: searchIter=" + searchIter);
}
final String searchTerm = "real" + random().nextInt(3);
final boolean fillFields = random().nextBoolean();
boolean getScores = random().nextBoolean();
final boolean getMaxScores = random().nextBoolean();
final Sort groupSort = getRandomSort();
//final Sort groupSort = new Sort(new SortField[] {new SortField("sort1", SortField.STRING), new SortField("id", SortField.INT)});
// TODO: also test null (= sort by relevance)
final Sort docSort = getRandomSort();
for(SortField sf : docSort.getSort()) {
if (sf.getType() == SortField.Type.SCORE) {
getScores = true;
break;
}
}
for(SortField sf : groupSort.getSort()) {
if (sf.getType() == SortField.Type.SCORE) {
getScores = true;
break;
}
}
final int topNGroups = TestUtil.nextInt(random(), 1, 30);
//final int topNGroups = 10;
final int docsPerGroup = TestUtil.nextInt(random(), 1, 50);
final int groupOffset = TestUtil.nextInt(random(), 0, (topNGroups - 1) / 2);
//final int groupOffset = 0;
final int docOffset = TestUtil.nextInt(random(), 0, docsPerGroup - 1);
//final int docOffset = 0;
final boolean doCache = random().nextBoolean();
final boolean doAllGroups = random().nextBoolean();
if (VERBOSE) {
System.out.println("TEST: groupSort=" + groupSort + " docSort=" + docSort + " searchTerm=" + searchTerm + " dF=" + r.docFreq(new Term("content", searchTerm)) +" dFBlock=" + rBlocks.docFreq(new Term("content", searchTerm)) + " topNGroups=" + topNGroups + " groupOffset=" + groupOffset + " docOffset=" + docOffset + " doCache=" + doCache + " docsPerGroup=" + docsPerGroup + " doAllGroups=" + doAllGroups + " getScores=" + getScores + " getMaxScores=" + getMaxScores);
}
String groupField = "group";
if (VERBOSE) {
System.out.println(" groupField=" + groupField);
}
final AbstractFirstPassGroupingCollector<?> c1 = createRandomFirstPassCollector(groupField, groupSort, groupOffset+topNGroups);
final CachingCollector cCache;
final Collector c;
final AbstractAllGroupsCollector<?> allGroupsCollector;
if (doAllGroups) {
allGroupsCollector = createAllGroupsCollector(c1, groupField);
} else {
allGroupsCollector = null;
}
final boolean useWrappingCollector = random().nextBoolean();
if (doCache) {
final double maxCacheMB = random().nextDouble();
if (VERBOSE) {
System.out.println("TEST: maxCacheMB=" + maxCacheMB);
}
if (useWrappingCollector) {
if (doAllGroups) {
cCache = CachingCollector.create(c1, true, maxCacheMB);
c = MultiCollector.wrap(cCache, allGroupsCollector);
} else {
c = cCache = CachingCollector.create(c1, true, maxCacheMB);
}
} else {
// Collect only into cache, then replay multiple times:
c = cCache = CachingCollector.create(false, true, maxCacheMB);
}
} else {
cCache = null;
if (doAllGroups) {
c = MultiCollector.wrap(c1, allGroupsCollector);
} else {
c = c1;
}
}
// Search top reader:
final Query query = new TermQuery(new Term("content", searchTerm));
s.search(query, c);
if (doCache && !useWrappingCollector) {
if (cCache.isCached()) {
// Replay for first-pass grouping
cCache.replay(c1);
if (doAllGroups) {
// Replay for all groups:
cCache.replay(allGroupsCollector);
}
} else {
// Replay by re-running search:
s.search(query, c1);
if (doAllGroups) {
s.search(query, allGroupsCollector);
}
}
}
// Get 1st pass top groups
final Collection<SearchGroup<BytesRef>> topGroups = getSearchGroups(c1, groupOffset, fillFields);
final TopGroups<BytesRef> groupsResult;
if (VERBOSE) {
System.out.println("TEST: first pass topGroups");
if (topGroups == null) {
System.out.println(" null");
} else {
for (SearchGroup<BytesRef> searchGroup : topGroups) {
System.out.println(" " + (searchGroup.groupValue == null ? "null" : searchGroup.groupValue) + ": " + Arrays.deepToString(searchGroup.sortValues));
}
}
}
// Get 1st pass top groups using shards
final TopGroups<BytesRef> topGroupsShards = searchShards(s, shards.subSearchers, query, groupSort, docSort,
groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, true, false);
final AbstractSecondPassGroupingCollector<?> c2;
if (topGroups != null) {
if (VERBOSE) {
System.out.println("TEST: topGroups");
for (SearchGroup<BytesRef> searchGroup : topGroups) {
System.out.println(" " + (searchGroup.groupValue == null ? "null" : searchGroup.groupValue.utf8ToString()) + ": " + Arrays.deepToString(searchGroup.sortValues));
}
}
c2 = createSecondPassCollector(c1, groupField, groupSort, docSort, groupOffset, docOffset + docsPerGroup, getScores, getMaxScores, fillFields);
if (doCache) {
if (cCache.isCached()) {
if (VERBOSE) {
System.out.println("TEST: cache is intact");
}
cCache.replay(c2);
} else {
if (VERBOSE) {
System.out.println("TEST: cache was too large");
}
s.search(query, c2);
}
} else {
s.search(query, c2);
}
if (doAllGroups) {
TopGroups<BytesRef> tempTopGroups = getTopGroups(c2, docOffset);
groupsResult = new TopGroups<>(tempTopGroups, allGroupsCollector.getGroupCount());
} else {
groupsResult = getTopGroups(c2, docOffset);
}
} else {
c2 = null;
groupsResult = null;
if (VERBOSE) {
System.out.println("TEST: no results");
}
}
final TopGroups<BytesRef> expectedGroups = slowGrouping(groupDocs, searchTerm, fillFields, getScores, getMaxScores, doAllGroups, groupSort, docSort, topNGroups, docsPerGroup, groupOffset, docOffset);
if (VERBOSE) {
if (expectedGroups == null) {
System.out.println("TEST: no expected groups");
} else {
System.out.println("TEST: expected groups totalGroupedHitCount=" + expectedGroups.totalGroupedHitCount);
for(GroupDocs<BytesRef> gd : expectedGroups.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits + " scoreDocs.len=" + gd.scoreDocs.length);
for(ScoreDoc sd : gd.scoreDocs) {
System.out.println(" id=" + sd.doc + " score=" + sd.score);
}
}
}
if (groupsResult == null) {
System.out.println("TEST: no matched groups");
} else {
System.out.println("TEST: matched groups totalGroupedHitCount=" + groupsResult.totalGroupedHitCount);
for(GroupDocs<BytesRef> gd : groupsResult.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
}
}
if (searchIter == 14) {
for(int docIDX=0;docIDX<s.getIndexReader().maxDoc();docIDX++) {
System.out.println("ID=" + docIDToID.get(docIDX) + " explain=" + s.explain(query, docIDX));
}
}
}
if (topGroupsShards == null) {
System.out.println("TEST: no matched-merged groups");
} else {
System.out.println("TEST: matched-merged groups totalGroupedHitCount=" + topGroupsShards.totalGroupedHitCount);
for(GroupDocs<BytesRef> gd : topGroupsShards.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
System.out.println(" id=" + docIDToID.get(sd.doc) + " score=" + sd.score);
}
}
}
}
assertEquals(docIDToID, expectedGroups, groupsResult, true, true, true, getScores, true);
// Confirm merged shards match:
assertEquals(docIDToID, expectedGroups, topGroupsShards, true, false, fillFields, getScores, true);
if (topGroupsShards != null) {
verifyShards(shards.docStarts, topGroupsShards);
}
final boolean needsScores = getScores || getMaxScores || docSort == null;
final BlockGroupingCollector c3 = new BlockGroupingCollector(groupSort, groupOffset+topNGroups, needsScores, lastDocInBlock);
final TermAllGroupsCollector allGroupsCollector2;
final Collector c4;
if (doAllGroups) {
// NOTE: must be "group" and not "group_dv"
// (groupField) because we didn't index doc
// values in the block index:
allGroupsCollector2 = new TermAllGroupsCollector("group");
c4 = MultiCollector.wrap(c3, allGroupsCollector2);
} else {
allGroupsCollector2 = null;
c4 = c3;
}
// Get block grouping result:
sBlocks.search(query, c4);
@SuppressWarnings({"unchecked","rawtypes"})
final TopGroups<BytesRef> tempTopGroupsBlocks = (TopGroups<BytesRef>) c3.getTopGroups(docSort, groupOffset, docOffset, docOffset+docsPerGroup, fillFields);
final TopGroups<BytesRef> groupsResultBlocks;
if (doAllGroups && tempTopGroupsBlocks != null) {
assertEquals((int) tempTopGroupsBlocks.totalGroupCount, allGroupsCollector2.getGroupCount());
groupsResultBlocks = new TopGroups<>(tempTopGroupsBlocks, allGroupsCollector2.getGroupCount());
} else {
groupsResultBlocks = tempTopGroupsBlocks;
}
if (VERBOSE) {
if (groupsResultBlocks == null) {
System.out.println("TEST: no block groups");
} else {
System.out.println("TEST: block groups totalGroupedHitCount=" + groupsResultBlocks.totalGroupedHitCount);
boolean first = true;
for(GroupDocs<BytesRef> gd : groupsResultBlocks.groups) {
System.out.println(" group=" + (gd.groupValue == null ? "null" : gd.groupValue.utf8ToString()) + " totalHits=" + gd.totalHits);
for(ScoreDoc sd : gd.scoreDocs) {
System.out.println(" id=" + docIDToIDBlocks.get(sd.doc) + " score=" + sd.score);
if (first) {
System.out.println("explain: " + sBlocks.explain(query, sd.doc));
first = false;
}
}
}
}
}
// Get shard'd block grouping result:
final TopGroups<BytesRef> topGroupsBlockShards = searchShards(sBlocks, shardsBlocks.subSearchers, query,
groupSort, docSort, groupOffset, topNGroups, docOffset, docsPerGroup, getScores, getMaxScores, false, false);
if (expectedGroups != null) {
// Fixup scores for reader2
for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
for(ScoreDoc hit : groupDocsHits.scoreDocs) {
final GroupDoc gd = groupDocsByID[hit.doc];
assertEquals(gd.id, hit.doc);
//System.out.println("fixup score " + hit.score + " to " + gd.score2 + " vs " + gd.score);
hit.score = gd.score2;
}
}
final SortField[] sortFields = groupSort.getSort();
final Map<Float,Float> termScoreMap = scoreMap.get(searchTerm);
for(int groupSortIDX=0;groupSortIDX<sortFields.length;groupSortIDX++) {
if (sortFields[groupSortIDX].getType() == SortField.Type.SCORE) {
for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
if (groupDocsHits.groupSortValues != null) {
//System.out.println("remap " + groupDocsHits.groupSortValues[groupSortIDX] + " to " + termScoreMap.get(groupDocsHits.groupSortValues[groupSortIDX]));
groupDocsHits.groupSortValues[groupSortIDX] = termScoreMap.get(groupDocsHits.groupSortValues[groupSortIDX]);
assertNotNull(groupDocsHits.groupSortValues[groupSortIDX]);
}
}
}
}
final SortField[] docSortFields = docSort.getSort();
for(int docSortIDX=0;docSortIDX<docSortFields.length;docSortIDX++) {
if (docSortFields[docSortIDX].getType() == SortField.Type.SCORE) {
for (GroupDocs<?> groupDocsHits : expectedGroups.groups) {
for(ScoreDoc _hit : groupDocsHits.scoreDocs) {
FieldDoc hit = (FieldDoc) _hit;
if (hit.fields != null) {
hit.fields[docSortIDX] = termScoreMap.get(hit.fields[docSortIDX]);
assertNotNull(hit.fields[docSortIDX]);
}
}
}
}
}
}
assertEquals(docIDToIDBlocks, expectedGroups, groupsResultBlocks, false, true, true, getScores, false);
assertEquals(docIDToIDBlocks, expectedGroups, topGroupsBlockShards, false, false, fillFields, getScores, false);
}
r.close();
dir.close();
rBlocks.close();
dirBlocks.close();
}

155
lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java
View File

@ -1,155 +0,0 @@
package org.apache.lucene.queries.function;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queries.function.valuesource.OrdFieldSource;
import org.apache.lucene.queries.function.valuesource.ReverseOrdFieldSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.junit.BeforeClass;
import org.junit.Test;
/**
* Test search based on OrdFieldSource and ReverseOrdFieldSource.
* <p/>
* Tests here create an index with a few documents, each having
* an indexed "id" field.
* The ord values of this field are later used for scoring.
* <p/>
* The order tests use Hits to verify that docs are ordered as expected.
* <p/>
* The exact score tests use TopDocs top to verify the exact score.
*/
public class TestOrdValues extends FunctionTestSetup {
@BeforeClass
public static void beforeClass() throws Exception {
createIndex(false);
}
/**
* Test OrdFieldSource
*/
@Test
public void testOrdFieldRank() throws Exception {
doTestRank(ID_FIELD, true);
}
/**
* Test ReverseOrdFieldSource
*/
@Test
public void testReverseOrdFieldRank() throws Exception {
doTestRank(ID_FIELD, false);
}
// Test that queries based on reverse/ordFieldScore scores correctly
private void doTestRank(String field, boolean inOrder) throws Exception {
IndexReader r = DirectoryReader.open(dir);
IndexSearcher s = newSearcher(r);
ValueSource vs;
if (inOrder) {
vs = new OrdFieldSource(field);
} else {
vs = new ReverseOrdFieldSource(field);
}
Query q = new FunctionQuery(vs);
log("test: " + q);
QueryUtils.check(random(), q, s);
ScoreDoc[] h = s.search(q, null, 1000).scoreDocs;
assertEquals("All docs should be matched!", N_DOCS, h.length);
String prevID = inOrder
? "IE" // greater than all ids of docs in this test ("ID0001", etc.)
: "IC"; // smaller than all ids of docs in this test ("ID0001", etc.)
for (int i = 0; i < h.length; i++) {
String resID = s.doc(h[i].doc).get(ID_FIELD);
log(i + ". score=" + h[i].score + " - " + resID);
log(s.explain(q, h[i].doc));
if (inOrder) {
assertTrue("res id " + resID + " should be < prev res id " + prevID, resID.compareTo(prevID) < 0);
} else {
assertTrue("res id " + resID + " should be > prev res id " + prevID, resID.compareTo(prevID) > 0);
}
prevID = resID;
}
r.close();
}
/**
* Test exact score for OrdFieldSource
*/
@Test
public void testOrdFieldExactScore() throws Exception {
doTestExactScore(ID_FIELD, true);
}
/**
* Test exact score for ReverseOrdFieldSource
*/
@Test
public void testReverseOrdFieldExactScore() throws Exception {
doTestExactScore(ID_FIELD, false);
}
// Test that queries based on reverse/ordFieldScore returns docs with expected score.
private void doTestExactScore(String field, boolean inOrder) throws Exception {
IndexReader r = DirectoryReader.open(dir);
IndexSearcher s = newSearcher(r);
ValueSource vs;
if (inOrder) {
vs = new OrdFieldSource(field);
} else {
vs = new ReverseOrdFieldSource(field);
}
Query q = new FunctionQuery(vs);
TopDocs td = s.search(q, null, 1000);
assertEquals("All docs should be matched!", N_DOCS, td.totalHits);
ScoreDoc sd[] = td.scoreDocs;
for (int i = 0; i < sd.length; i++) {
float score = sd[i].score;
String id = s.getIndexReader().document(sd[i].doc).get(ID_FIELD);
log("-------- " + i + ". Explain doc " + id);
log(s.explain(q, sd[i].doc));
float expectedScore = N_DOCS - i - 1;
assertEquals("score of result " + i + " shuould be " + expectedScore + " != " + score, expectedScore, score, TEST_SCORE_TOLERANCE_DELTA);
String expectedId = inOrder
? id2String(N_DOCS - i) // in-order ==> larger values first
: id2String(i + 1); // reverse ==> smaller values first
assertTrue("id of result " + i + " shuould be " + expectedId + " != " + score, expectedId.equals(id));
}
r.close();
}
// LUCENE-1250
public void testEqualsNull() throws Exception {
OrdFieldSource ofs = new OrdFieldSource("f");
assertFalse(ofs.equals(null));
ReverseOrdFieldSource rofs = new ReverseOrdFieldSource("f");
assertFalse(rofs.equals(null));
}
}

4
lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java
View File

@ -21,7 +21,7 @@ import java.text.Collator;
import org.apache.lucene.search.MultiTermQueryWrapperFilter;
import org.apache.lucene.search.NumericRangeFilter; // javadoc
import org.apache.lucene.search.FieldCacheRangeFilter; // javadoc
import org.apache.lucene.search.DocValuesRangeFilter; // javadoc
/**
* A Filter that restricts search results to a range of term
@ -33,7 +33,7 @@ import org.apache.lucene.search.FieldCacheRangeFilter; // javadoc
* for numerical ranges; use {@link NumericRangeFilter} instead.
*
* <p>If you construct a large number of range filters with different ranges but on the
* same field, {@link FieldCacheRangeFilter} may have significantly better performance.
* same field, {@link DocValuesRangeFilter} may have significantly better performance.
* @deprecated Index collation keys with CollationKeyAnalyzer or ICUCollationKeyAnalyzer instead.
* This class will be removed in Lucene 5.0
*/

6
lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
View File

@ -130,12 +130,6 @@ public class QueryUtils {
}
}
public static void purgeFieldCache(IndexReader r) throws IOException {
// this is just a hack, to get an atomic reader that contains all subreaders for insanity checks
// nocommit: WTF? nuke this shit!
// FieldCache.DEFAULT.purgeByCacheKey(SlowCompositeReaderWrapper.wrap(r).getCoreCacheKey());
}
/** This is a MultiReader that can be used for randomly wrapping other readers
* without creating FieldCache insanity.
* The trick is to use an opaque/fake cache key. */

42
lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -627,7 +627,6 @@ public abstract class LuceneTestCase extends Assert {
.around(threadAndTestNameRule)
.around(new SystemPropertiesInvariantRule(IGNORED_INVARIANT_PROPERTIES))
.around(new TestRuleSetupAndRestoreInstanceEnv())
.around(new TestRuleFieldCacheSanity())
.around(parentChainCallRule);
private static final Map<String,FieldType> fieldToType = new HashMap<String,FieldType>();
@ -738,47 +737,6 @@ public abstract class LuceneTestCase extends Assert {
return Thread.currentThread() == threadAndTestNameRule.testCaseThread;
}
// nocommit: move to SolrTestCaseJ4 ?
/**
* Asserts that FieldCacheSanityChecker does not detect any
* problems with FieldCache.DEFAULT.
* <p>
* If any problems are found, they are logged to System.err
* (allong with the msg) when the Assertion is thrown.
* </p>
* <p>
* This method is called by tearDown after every test method,
* however IndexReaders scoped inside test methods may be garbage
* collected prior to this method being called, causing errors to
* be overlooked. Tests are encouraged to keep their IndexReaders
* scoped at the class level, or to explicitly call this method
* directly in the same scope as the IndexReader.
* </p>
*/
protected static void assertSaneFieldCaches(final String msg) {
/* final CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries();
Insanity[] insanity = null;
try {
try {
insanity = FieldCacheSanityChecker.checkSanity(entries);
} catch (RuntimeException e) {
dumpArray(msg + ": FieldCache", entries, System.err);
throw e;
}
assertEquals(msg + ": Insane FieldCache usage(s) found",
0, insanity.length);
insanity = null;
} finally {
// report this in the event of any exception/failure
// if no failure, then insanity will be null anyway
if (null != insanity) {
dumpArray(msg + ": Insane FieldCache usage(s)", insanity, System.err);
}
} */
}
/**
* Returns a number of at least <code>i</code>
* <p>

66
lucene/test-framework/src/java/org/apache/lucene/util/TestRuleFieldCacheSanity.java
View File

@ -1,66 +0,0 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
/**
* This rule will fail the test if it has insane field caches.
* <p>
* calling assertSaneFieldCaches here isn't as useful as having test
* classes call it directly from the scope where the index readers
* are used, because they could be gc'ed just before this tearDown
* method is called.
* <p>
* But it's better then nothing.
* <p>
* If you are testing functionality that you know for a fact
* "violates" FieldCache sanity, then you should either explicitly
* call purgeFieldCache at the end of your test method, or refactor
* your Test class so that the inconsistent FieldCache usages are
* isolated in distinct test methods
*/
public class TestRuleFieldCacheSanity implements TestRule {
// nocommit: move to solr?
@Override
public Statement apply(final Statement s, final Description d) {
return new Statement() {
@Override
public void evaluate() throws Throwable {
s.evaluate();
Throwable problem = null;
try {
LuceneTestCase.assertSaneFieldCaches(d.getDisplayName());
} catch (Throwable t) {
problem = t;
}
//FieldCache.DEFAULT.purgeAllCaches();
if (problem != null) {
Rethrow.rethrow(problem);
}
}
};
}
}

4
solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
View File

@ -34,7 +34,7 @@ import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocTermOrdsRangeFilter;
import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.DocValuesRangeFilter;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermRangeQuery;
@ -280,7 +280,7 @@ public class ICUCollationField extends FieldType {
return new ConstantScoreQuery(DocTermOrdsRangeFilter.newBytesRefRange(
field.getName(), low, high, minInclusive, maxInclusive));
} else {
return new ConstantScoreQuery(FieldCacheRangeFilter.newBytesRefRange(
return new ConstantScoreQuery(DocValuesRangeFilter.newBytesRefRange(
field.getName(), low, high, minInclusive, maxInclusive));
}
} else {

2
solr/core/src/java/org/apache/solr/schema/BoolField.java
View File

@ -32,7 +32,6 @@ import org.apache.lucene.index.StorableField;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.BoolDocValues;
import org.apache.lucene.queries.function.valuesource.OrdFieldSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
@ -42,6 +41,7 @@ import org.apache.lucene.util.mutable.MutableValueBool;
import org.apache.solr.analysis.SolrAnalyzer;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.function.OrdFieldSource;
/**
*
*/

4
solr/core/src/java/org/apache/solr/schema/CollationField.java
View File

@ -38,7 +38,7 @@ import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocTermOrdsRangeFilter;
import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.DocValuesRangeFilter;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermRangeQuery;
@ -255,7 +255,7 @@ public class CollationField extends FieldType {
return new ConstantScoreQuery(DocTermOrdsRangeFilter.newBytesRefRange(
field.getName(), low, high, minInclusive, maxInclusive));
} else {
return new ConstantScoreQuery(FieldCacheRangeFilter.newBytesRefRange(
return new ConstantScoreQuery(DocValuesRangeFilter.newBytesRefRange(
field.getName(), low, high, minInclusive, maxInclusive));
}
} else {

2
solr/core/src/java/org/apache/solr/schema/EnumField.java
View File

@ -240,7 +240,7 @@ public class EnumField extends PrimitiveFieldType {
Query query = null;
final boolean matchOnly = field.hasDocValues() && !field.indexed();
if (matchOnly) {
query = new ConstantScoreQuery(FieldCacheRangeFilter.newIntRange(field.getName(),
query = new ConstantScoreQuery(DocValuesRangeFilter.newIntRange(field.getName(),
min == null ? null : minValue,
max == null ? null : maxValue,
minInclusive, maxInclusive));

8
solr/core/src/java/org/apache/solr/schema/FieldType.java
View File

@ -32,8 +32,8 @@ import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocTermOrdsRangeFilter;
import org.apache.lucene.search.DocTermOrdsRewriteMethod;
import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.FieldCacheRewriteMethod;
import org.apache.lucene.search.DocValuesRangeFilter;
import org.apache.lucene.search.DocValuesRewriteMethod;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
@ -698,7 +698,7 @@ public abstract class FieldType extends FieldProperties {
part2 == null ? null : new BytesRef(toInternal(part2)),
minInclusive, maxInclusive));
} else {
return new ConstantScoreQuery(FieldCacheRangeFilter.newStringRange(
return new ConstantScoreQuery(DocValuesRangeFilter.newStringRange(
field.getName(),
part1 == null ? null : toInternal(part1),
part2 == null ? null : toInternal(part2),
@ -742,7 +742,7 @@ public abstract class FieldType extends FieldProperties {
*/
public MultiTermQuery.RewriteMethod getRewriteMethod(QParser parser, SchemaField field) {
if (!field.indexed() && field.hasDocValues()) {
return field.multiValued() ? new DocTermOrdsRewriteMethod() : new FieldCacheRewriteMethod();
return field.multiValued() ? new DocTermOrdsRewriteMethod() : new DocValuesRewriteMethod();
} else {
return MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT;
}

12
solr/core/src/java/org/apache/solr/schema/TrieField.java
View File

@ -39,7 +39,7 @@ import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.DocValuesRangeFilter;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
@ -307,7 +307,7 @@ public class TrieField extends PrimitiveFieldType {
switch (type) {
case INTEGER:
if (matchOnly) {
query = new ConstantScoreQuery(FieldCacheRangeFilter.newIntRange(field.getName(),
query = new ConstantScoreQuery(DocValuesRangeFilter.newIntRange(field.getName(),
min == null ? null : Integer.parseInt(min),
max == null ? null : Integer.parseInt(max),
minInclusive, maxInclusive));
@ -320,7 +320,7 @@ public class TrieField extends PrimitiveFieldType {
break;
case FLOAT:
if (matchOnly) {
query = new ConstantScoreQuery(FieldCacheRangeFilter.newFloatRange(field.getName(),
query = new ConstantScoreQuery(DocValuesRangeFilter.newFloatRange(field.getName(),
min == null ? null : Float.parseFloat(min),
max == null ? null : Float.parseFloat(max),
minInclusive, maxInclusive));
@ -333,7 +333,7 @@ public class TrieField extends PrimitiveFieldType {
break;
case LONG:
if (matchOnly) {
query = new ConstantScoreQuery(FieldCacheRangeFilter.newLongRange(field.getName(),
query = new ConstantScoreQuery(DocValuesRangeFilter.newLongRange(field.getName(),
min == null ? null : Long.parseLong(min),
max == null ? null : Long.parseLong(max),
minInclusive, maxInclusive));
@ -346,7 +346,7 @@ public class TrieField extends PrimitiveFieldType {
break;
case DOUBLE:
if (matchOnly) {
query = new ConstantScoreQuery(FieldCacheRangeFilter.newDoubleRange(field.getName(),
query = new ConstantScoreQuery(DocValuesRangeFilter.newDoubleRange(field.getName(),
min == null ? null : Double.parseDouble(min),
max == null ? null : Double.parseDouble(max),
minInclusive, maxInclusive));
@ -359,7 +359,7 @@ public class TrieField extends PrimitiveFieldType {
break;
case DATE:
if (matchOnly) {
query = new ConstantScoreQuery(FieldCacheRangeFilter.newLongRange(field.getName(),
query = new ConstantScoreQuery(DocValuesRangeFilter.newLongRange(field.getName(),
min == null ? null : dateField.parseMath(null, min).getTime(),
max == null ? null : dateField.parseMath(null, max).getTime(),
minInclusive, maxInclusive));

1
solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -193,7 +193,6 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
// this reader supports reopen
private static DirectoryReader wrapReader(SolrCore core, DirectoryReader reader) {
assert reader != null;
assert !reader.getClass().getSimpleName().startsWith("Uninverting"); // nocommit
return UninvertingReader.wrap(reader, core.getLatestSchema().getUninversionMap(reader));
}

2
solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
View File

@ -42,6 +42,8 @@ import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.*;
import org.apache.solr.search.function.CollapseScoreFunction;
import org.apache.solr.search.function.OrdFieldSource;
import org.apache.solr.search.function.ReverseOrdFieldSource;
import org.apache.solr.search.function.distance.*;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;

20
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java → solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java
View File

@ -15,14 +15,13 @@
* limitations under the License.
*/
package org.apache.lucene.queries.function.valuesource;
package org.apache.solr.search.function;
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.CompositeReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.ReaderUtil;
@ -31,8 +30,10 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueInt;
import org.apache.solr.search.SolrIndexSearcher;
/**
* Obtains the ordinal of the field value from {@link AtomicReader#getSortedDocValues}.
@ -65,13 +66,20 @@ public class OrdFieldSource extends ValueSource {
}
// TODO: this is trappy? perhaps this query instead should make you pass a slow reader yourself?
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final int off = readerContext.docBase;
final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
final AtomicReader r = SlowCompositeReaderWrapper.wrap(topReader);
final SortedDocValues sindex = DocValues.getSorted(r, field);
final AtomicReader r;
Object o = context.get("searcher");
if (o instanceof SolrIndexSearcher) {
// reuse ordinalmap
r = ((SolrIndexSearcher)o).getAtomicReader();
} else {
IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
r = SlowCompositeReaderWrapper.wrap(topReader);
}
// if its e.g. tokenized/multivalued, emulate old behavior of single-valued fc
final SortedDocValues sindex = SortedSetSelector.wrap(DocValues.getSortedSet(r, field), SortedSetSelector.Type.MIN);
return new IntDocValues(this) {
protected String toTerm(String readableValue) {
return readableValue;

20
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java → solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.lucene.queries.function.valuesource;
package org.apache.solr.search.function;
import java.io.IOException;
import java.util.Map;
@ -31,6 +31,8 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.solr.search.SolrIndexSearcher;
/**
* Obtains the ordinal of the field value from {@link AtomicReader#getSortedDocValues}
@ -65,14 +67,20 @@ public class ReverseOrdFieldSource extends ValueSource {
return "rord("+field+')';
}
// TODO: this is trappy? perhaps this query instead should make you pass a slow reader yourself?
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
final AtomicReader r = SlowCompositeReaderWrapper.wrap(topReader);
final int off = readerContext.docBase;
final SortedDocValues sindex = DocValues.getSorted(r, field);
final AtomicReader r;
Object o = context.get("searcher");
if (o instanceof SolrIndexSearcher) {
// reuse ordinalmap
r = ((SolrIndexSearcher)o).getAtomicReader();
} else {
IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
r = SlowCompositeReaderWrapper.wrap(topReader);
}
// if its e.g. tokenized/multivalued, emulate old behavior of single-valued fc
final SortedDocValues sindex = SortedSetSelector.wrap(DocValues.getSortedSet(r, field), SortedSetSelector.Type.MIN);
final int end = sindex.getValueCount();
return new IntDocValues(this) {

3
solr/core/src/test/org/apache/solr/TestDistributedSearch.java
View File

@ -421,9 +421,6 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
// Thread.sleep(10000000000L);
// nocommit: split test if needed
// FieldCache.DEFAULT.purgeAllCaches(); // hide FC insanity
del("*:*"); // delete all docs and test stats request
commit();
try {

2
solr/core/src/test/org/apache/solr/TestGroupingSearch.java
View File

@ -517,8 +517,6 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
,"/grouped/"+f+"/matches==10"
,"/facet_counts/facet_fields/"+f+"==['1',3, '2',3, '3',2, '4',1, '5',1]"
);
// nocommit: split test if needed
// FieldCache.DEFAULT.purgeAllCaches(); // hide FC insanity
// test that grouping works with highlighting
assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id"

37
solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java
View File

@ -113,28 +113,23 @@ public class TestRandomDVFaceting extends SolrTestCaseJ4 {
@Test
public void testRandomFaceting() throws Exception {
try {
Random rand = random();
int iter = atLeast(100);
init();
addMoreDocs(0);
for (int i=0; i<iter; i++) {
doFacetTests();
if (rand.nextInt(100) < 5) {
init();
}
addMoreDocs(rand.nextInt(indexSize) + 1);
if (rand.nextInt(100) < 50) {
deleteSomeDocs();
}
Random rand = random();
int iter = atLeast(100);
init();
addMoreDocs(0);
for (int i=0; i<iter; i++) {
doFacetTests();
if (rand.nextInt(100) < 5) {
init();
}
addMoreDocs(rand.nextInt(indexSize) + 1);
if (rand.nextInt(100) < 50) {
deleteSomeDocs();
}
} finally {
// nocommit: split test if needed
// FieldCache.DEFAULT.purgeAllCaches(); // hide FC insanity
}
}

37
solr/core/src/test/org/apache/solr/TestRandomFaceting.java
View File

@ -112,28 +112,23 @@ public class TestRandomFaceting extends SolrTestCaseJ4 {
@Test
public void testRandomFaceting() throws Exception {
try {
Random rand = random();
int iter = atLeast(100);
init();
addMoreDocs(0);
for (int i=0; i<iter; i++) {
doFacetTests();
if (rand.nextInt(100) < 5) {
init();
}
addMoreDocs(rand.nextInt(indexSize) + 1);
if (rand.nextInt(100) < 50) {
deleteSomeDocs();
}
Random rand = random();
int iter = atLeast(100);
init();
addMoreDocs(0);
for (int i=0; i<iter; i++) {
doFacetTests();
if (rand.nextInt(100) < 5) {
init();
}
addMoreDocs(rand.nextInt(indexSize) + 1);
if (rand.nextInt(100) < 50) {
deleteSomeDocs();
}
} finally {
// nocommit: split test if needed
// FieldCache.DEFAULT.purgeAllCaches(); // hide FC insanity
}
}

18
solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
View File

@ -201,9 +201,6 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
singleTest(field,"sum(query($v1,5),query($v1,7))",
Arrays.asList("v1","\0:[* TO *]"), 88,12
);
// nocommit: split test if needed
// FieldCache.DEFAULT.purgeAllCaches(); // hide FC insanity
}
@Test
@ -283,10 +280,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
singleTest(field, "\0", answers);
// System.out.println("Done test "+i);
}
// nocommit: split test if needed
// FieldCache.DEFAULT.purgeAllCaches(); // hide FC insanity
}
}
@Test
@ -642,10 +636,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
assertU(adoc("id", "10000")); // will get same reader if no index change
assertU(commit());
singleTest(fieldAsFunc, "sqrt(\0)");
assertTrue(orig != FileFloatSource.onlyForTesting);
// nocommit: split test if needed
// FieldCache.DEFAULT.purgeAllCaches(); // hide FC insanity
assertTrue(orig != FileFloatSource.onlyForTesting);
}
/**
@ -670,10 +661,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
100,100, -4,-4, 0,0, 10,10, 25,25, 5,5, 77,77, 1,1);
singleTest(fieldAsFunc, "sqrt(\0)",
100,10, 25,5, 0,0, 1,1);
singleTest(fieldAsFunc, "log(\0)", 1,0);
// nocommit: split test if needed
// FieldCache.DEFAULT.purgeAllCaches(); // hide FC insanity
singleTest(fieldAsFunc, "log(\0)", 1,0);
}
@Test

310
solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java Normal file
View File

@ -0,0 +1,310 @@
package org.apache.solr.search.function;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
/**
* Test search based on OrdFieldSource and ReverseOrdFieldSource.
* <p/>
* Tests here create an index with a few documents, each having
* an indexed "id" field.
* The ord values of this field are later used for scoring.
* <p/>
* The order tests use Hits to verify that docs are ordered as expected.
* <p/>
* The exact score tests use TopDocs top to verify the exact score.
*/
public class TestOrdValues extends LuceneTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
createIndex(false);
}
/**
* Test OrdFieldSource
*/
@Test
public void testOrdFieldRank() throws Exception {
doTestRank(ID_FIELD, true);
}
/**
* Test ReverseOrdFieldSource
*/
@Test
public void testReverseOrdFieldRank() throws Exception {
doTestRank(ID_FIELD, false);
}
// Test that queries based on reverse/ordFieldScore scores correctly
private void doTestRank(String field, boolean inOrder) throws Exception {
IndexReader r = DirectoryReader.open(dir);
IndexSearcher s = newSearcher(r);
ValueSource vs;
if (inOrder) {
vs = new OrdFieldSource(field);
} else {
vs = new ReverseOrdFieldSource(field);
}
Query q = new FunctionQuery(vs);
log("test: " + q);
QueryUtils.check(random(), q, s);
ScoreDoc[] h = s.search(q, null, 1000).scoreDocs;
assertEquals("All docs should be matched!", N_DOCS, h.length);
String prevID = inOrder
? "IE" // greater than all ids of docs in this test ("ID0001", etc.)
: "IC"; // smaller than all ids of docs in this test ("ID0001", etc.)
for (int i = 0; i < h.length; i++) {
String resID = s.doc(h[i].doc).get(ID_FIELD);
log(i + ". score=" + h[i].score + " - " + resID);
log(s.explain(q, h[i].doc));
if (inOrder) {
assertTrue("res id " + resID + " should be < prev res id " + prevID, resID.compareTo(prevID) < 0);
} else {
assertTrue("res id " + resID + " should be > prev res id " + prevID, resID.compareTo(prevID) > 0);
}
prevID = resID;
}
r.close();
}
/**
* Test exact score for OrdFieldSource
*/
@Test
public void testOrdFieldExactScore() throws Exception {
doTestExactScore(ID_FIELD, true);
}
/**
* Test exact score for ReverseOrdFieldSource
*/
@Test
public void testReverseOrdFieldExactScore() throws Exception {
doTestExactScore(ID_FIELD, false);
}
// Test that queries based on reverse/ordFieldScore returns docs with expected score.
private void doTestExactScore(String field, boolean inOrder) throws Exception {
IndexReader r = DirectoryReader.open(dir);
IndexSearcher s = newSearcher(r);
ValueSource vs;
if (inOrder) {
vs = new OrdFieldSource(field);
} else {
vs = new ReverseOrdFieldSource(field);
}
Query q = new FunctionQuery(vs);
TopDocs td = s.search(q, null, 1000);
assertEquals("All docs should be matched!", N_DOCS, td.totalHits);
ScoreDoc sd[] = td.scoreDocs;
for (int i = 0; i < sd.length; i++) {
float score = sd[i].score;
String id = s.getIndexReader().document(sd[i].doc).get(ID_FIELD);
log("-------- " + i + ". Explain doc " + id);
log(s.explain(q, sd[i].doc));
float expectedScore = N_DOCS - i - 1;
assertEquals("score of result " + i + " shuould be " + expectedScore + " != " + score, expectedScore, score, TEST_SCORE_TOLERANCE_DELTA);
String expectedId = inOrder
? id2String(N_DOCS - i) // in-order ==> larger values first
: id2String(i + 1); // reverse ==> smaller values first
assertTrue("id of result " + i + " shuould be " + expectedId + " != " + score, expectedId.equals(id));
}
r.close();
}
// LUCENE-1250
public void testEqualsNull() throws Exception {
OrdFieldSource ofs = new OrdFieldSource("f");
assertFalse(ofs.equals(null));
ReverseOrdFieldSource rofs = new ReverseOrdFieldSource("f");
assertFalse(rofs.equals(null));
}
/**
* Actual score computation order is slightly different than assumptios
* this allows for a small amount of variation
*/
protected static float TEST_SCORE_TOLERANCE_DELTA = 0.001f;
protected static final int N_DOCS = 17; // select a primary number > 2
protected static final String ID_FIELD = "id";
protected static final String TEXT_FIELD = "text";
protected static final String INT_FIELD = "iii";
protected static final String FLOAT_FIELD = "fff";
protected ValueSource INT_VALUESOURCE = new IntFieldSource(INT_FIELD);
protected ValueSource FLOAT_VALUESOURCE = new FloatFieldSource(FLOAT_FIELD);
private static final String DOC_TEXT_LINES[] = {
"Well, this is just some plain text we use for creating the ",
"test documents. It used to be a text from an online collection ",
"devoted to first aid, but if there was there an (online) lawyers ",
"first aid collection with legal advices, \"it\" might have quite ",
"probably advised one not to include \"it\"'s text or the text of ",
"any other online collection in one's code, unless one has money ",
"that one don't need and one is happy to donate for lawyers ",
"charity. Anyhow at some point, rechecking the usage of this text, ",
"it became uncertain that this text is free to use, because ",
"the web site in the disclaimer of he eBook containing that text ",
"was not responding anymore, and at the same time, in projGut, ",
"searching for first aid no longer found that eBook as well. ",
"So here we are, with a perhaps much less interesting ",
"text for the test, but oh much much safer. ",
};
protected static Directory dir;
protected static Analyzer anlzr;
@AfterClass
public static void afterClassFunctionTestSetup() throws Exception {
dir.close();
dir = null;
anlzr = null;
}
protected static void createIndex(boolean doMultiSegment) throws Exception {
if (VERBOSE) {
System.out.println("TEST: setUp");
}
// prepare a small index with just a few documents.
dir = newDirectory();
anlzr = new MockAnalyzer(random());
IndexWriterConfig iwc = newIndexWriterConfig( TEST_VERSION_CURRENT, anlzr).setMergePolicy(newLogMergePolicy());
if (doMultiSegment) {
iwc.setMaxBufferedDocs(TestUtil.nextInt(random(), 2, 7));
}
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
// add docs not exactly in natural ID order, to verify we do check the order of docs by scores
int remaining = N_DOCS;
boolean done[] = new boolean[N_DOCS];
int i = 0;
while (remaining > 0) {
if (done[i]) {
throw new Exception("to set this test correctly N_DOCS=" + N_DOCS + " must be primary and greater than 2!");
}
addDoc(iw, i);
done[i] = true;
i = (i + 4) % N_DOCS;
remaining --;
}
if (!doMultiSegment) {
if (VERBOSE) {
System.out.println("TEST: setUp full merge");
}
iw.forceMerge(1);
}
iw.shutdown();
if (VERBOSE) {
System.out.println("TEST: setUp done close");
}
}
private static void addDoc(RandomIndexWriter iw, int i) throws Exception {
Document d = new Document();
Field f;
int scoreAndID = i + 1;
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.setTokenized(false);
customType.setOmitNorms(true);
f = newField(ID_FIELD, id2String(scoreAndID), customType); // for debug purposes
d.add(f);
d.add(new SortedDocValuesField(ID_FIELD, new BytesRef(id2String(scoreAndID))));
FieldType customType2 = new FieldType(TextField.TYPE_NOT_STORED);
customType2.setOmitNorms(true);
f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search
d.add(f);
f = new IntField(INT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
d.add(new NumericDocValuesField(INT_FIELD, scoreAndID));
f = new FloatField(FLOAT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
d.add(new NumericDocValuesField(FLOAT_FIELD, Float.floatToRawIntBits(scoreAndID)));
iw.addDocument(d);
log("added: " + d);
}
// 17 --> ID00017
protected static String id2String(int scoreAndID) {
String s = "000000000" + scoreAndID;
int n = ("" + N_DOCS).length() + 3;
int k = s.length() - n;
return "ID" + s.substring(k);
}
// some text line for regular search
private static String textLine(int docNum) {
return DOC_TEXT_LINES[docNum % DOC_TEXT_LINES.length];
}
// extract expected doc score from its ID Field: "ID7" --> 7.0
protected static float expectedFieldScore(String docIDFieldVal) {
return Float.parseFloat(docIDFieldVal.substring(2));
}
// debug messages (change DBG to true for anything to print)
protected static void log(Object o) {
if (VERBOSE) {
System.out.println(o.toString());
}
}
}

2
solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
View File

@ -283,8 +283,6 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
@Override
public void tearDown() throws Exception {
destroyServers();
// nocommit: split test if needed
// FieldCache.DEFAULT.purgeAllCaches(); // hide FC insanity
super.tearDown();
}