mirror of https://github.com/apache/lucene.git
LUCENE-7714: Add a range query in sandbox that takes advantage of index sorting.
This commit is contained in:
Julie Tibshirani
jimczi
parent
583c219183
commit
5bf023cf19
|
|
@ -65,6 +65,9 @@ New Features
|
||||||
in an efficient manner, for use as a monitoring or classification tool.
|
in an efficient manner, for use as a monitoring or classification tool.
|
||||||
(Alan Woodward)
|
(Alan Woodward)
|
||||||
|
|
||||||
|
* LUCENE-7714: Add a numeric range query in sandbox that takes advantage of index sorting.
|
||||||
|
(Julie Tibshirani via Jim Ferenczi)
|
||||||
|
|
||||||
Bug Fixes
|
Bug Fixes
|
||||||
|
|
||||||
* LUCENE-8831: Fixed LatLonShapeBoundingBoxQuery .hashCode methods. (Ignacio Vera)
|
* LUCENE-8831: Fixed LatLonShapeBoundingBoxQuery .hashCode methods. (Ignacio Vera)
|
||||||
|
|
@ -445,7 +448,7 @@ Changes in Runtime Behavior
|
||||||
|
|
||||||
* LUCENE-8535: Highlighter and FVH doesn't support ToParent and ToChildBlockJoinQuery out of the
|
* LUCENE-8535: Highlighter and FVH doesn't support ToParent and ToChildBlockJoinQuery out of the
|
||||||
box anymore. In order to highlight on Block-Join Queries a custom WeightedSpanTermExtractor / FieldQuery
|
box anymore. In order to highlight on Block-Join Queries a custom WeightedSpanTermExtractor / FieldQuery
|
||||||
should be used. (Simon Willnauer, Jim Ferenczi, Julie Tibshiran)
|
should be used. (Simon Willnauer, Jim Ferenczi, Julie Tibshirani)
|
||||||
|
|
||||||
* LUCENE-8563: BM25 scores don't include the (k1+1) factor in their numerator
|
* LUCENE-8563: BM25 scores don't include the (k1+1) factor in their numerator
|
||||||
anymore. This doesn't affect ordering as this is a constant factor which is
|
anymore. This doesn't affect ordering as this is a constant factor which is
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,301 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.lucene.search;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
import org.apache.lucene.index.DocValues;
|
||||||
|
import org.apache.lucene.index.IndexReader;
|
||||||
|
import org.apache.lucene.index.LeafReaderContext;
|
||||||
|
import org.apache.lucene.index.NumericDocValues;
|
||||||
|
import org.apache.lucene.index.SortedNumericDocValues;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A range query that can take advantage of the fact that the index is sorted to speed up
|
||||||
|
* execution. If the index is sorted on the same field as the query, it performs binary
|
||||||
|
* search on the field's numeric doc values to find the documents at the lower and upper
|
||||||
|
* ends of the range.
|
||||||
|
*
|
||||||
|
* This optimized execution strategy is only used if the following conditions hold:
|
||||||
|
* <ul>
|
||||||
|
* <li> The index is sorted, and its primary sort is on the same field as the query.
|
||||||
|
* <li> The query field has either {@link SortedNumericDocValues} or {@link NumericDocValues}.
|
||||||
|
* <li> The segments must have at most one field value per document (otherwise we cannot easily
|
||||||
|
* determine the matching document IDs through a binary search).
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* If any of these conditions isn't met, the search is delegated to {@code fallbackQuery}.
|
||||||
|
*
|
||||||
|
* This fallback must be an equivalent range query -- it should produce the same documents and give
|
||||||
|
* constant scores. As an example, an {@link IndexSortSortedNumericDocValuesRangeQuery} might be
|
||||||
|
* constructed as follows:
|
||||||
|
* <pre class="prettyprint">
|
||||||
|
* String field = "field";
|
||||||
|
* long lowerValue = 0, long upperValue = 10;
|
||||||
|
* Query fallbackQuery = LongPoint.newRangeQuery(field, lowerValue, upperValue);
|
||||||
|
* Query rangeQuery = new IndexSortSortedNumericDocValuesRangeQuery(
|
||||||
|
* field, lowerValue, upperValue, fallbackQuery);
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @lucene.experimental
|
||||||
|
*/
|
||||||
|
public class IndexSortSortedNumericDocValuesRangeQuery extends Query {
|
||||||
|
|
||||||
|
private final String field;
|
||||||
|
private final long lowerValue;
|
||||||
|
private final long upperValue;
|
||||||
|
private final Query fallbackQuery;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new {@link IndexSortSortedNumericDocValuesRangeQuery}.
|
||||||
|
*
|
||||||
|
* @param field The field name.
|
||||||
|
* @param lowerValue The lower end of the range (inclusive).
|
||||||
|
* @param upperValue The upper end of the range (exclusive).
|
||||||
|
* @param fallbackQuery A query to fall back to if the optimization cannot be applied.
|
||||||
|
*/
|
||||||
|
public IndexSortSortedNumericDocValuesRangeQuery(String field,
|
||||||
|
long lowerValue,
|
||||||
|
long upperValue,
|
||||||
|
Query fallbackQuery) {
|
||||||
|
this.field = Objects.requireNonNull(field);
|
||||||
|
this.lowerValue = lowerValue;
|
||||||
|
this.upperValue = upperValue;
|
||||||
|
this.fallbackQuery = fallbackQuery;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Query getFallbackQuery() {
|
||||||
|
return fallbackQuery;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (this == o) return true;
|
||||||
|
if (o == null || getClass() != o.getClass()) return false;
|
||||||
|
IndexSortSortedNumericDocValuesRangeQuery that = (IndexSortSortedNumericDocValuesRangeQuery) o;
|
||||||
|
return lowerValue == that.lowerValue &&
|
||||||
|
upperValue == that.upperValue &&
|
||||||
|
Objects.equals(field, that.field) &&
|
||||||
|
Objects.equals(fallbackQuery, that.fallbackQuery);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return Objects.hash(field, lowerValue, upperValue, fallbackQuery);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void visit(QueryVisitor visitor) {
|
||||||
|
if (visitor.acceptField(field)) {
|
||||||
|
visitor.visitLeaf(this);
|
||||||
|
fallbackQuery.visit(visitor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString(String field) {
|
||||||
|
StringBuilder b = new StringBuilder();
|
||||||
|
if (this.field.equals(field) == false) {
|
||||||
|
b.append(this.field).append(":");
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
.append("[")
|
||||||
|
.append(lowerValue)
|
||||||
|
.append(" TO ")
|
||||||
|
.append(upperValue)
|
||||||
|
.append("]")
|
||||||
|
.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Query rewrite(IndexReader reader) throws IOException {
|
||||||
|
if (lowerValue == Long.MIN_VALUE && upperValue == Long.MAX_VALUE) {
|
||||||
|
return new DocValuesFieldExistsQuery(field);
|
||||||
|
}
|
||||||
|
|
||||||
|
Query rewrittenFallback = fallbackQuery.rewrite(reader);
|
||||||
|
if (rewrittenFallback == fallbackQuery) {
|
||||||
|
return this;
|
||||||
|
} else {
|
||||||
|
return new IndexSortSortedNumericDocValuesRangeQuery(
|
||||||
|
field, lowerValue, upperValue, rewrittenFallback);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
|
||||||
|
Weight fallbackWeight = fallbackQuery.createWeight(searcher, scoreMode, boost);
|
||||||
|
|
||||||
|
return new ConstantScoreWeight(this, boost) {
|
||||||
|
@Override
|
||||||
|
public Scorer scorer(LeafReaderContext context) throws IOException {
|
||||||
|
SortedNumericDocValues sortedNumericValues = DocValues.getSortedNumeric(context.reader(), field);
|
||||||
|
NumericDocValues numericValues = DocValues.unwrapSingleton(sortedNumericValues);
|
||||||
|
|
||||||
|
if (numericValues != null) {
|
||||||
|
Sort indexSort = context.reader().getMetaData().getSort();
|
||||||
|
if (indexSort != null
|
||||||
|
&& indexSort.getSort().length > 0
|
||||||
|
&& indexSort.getSort()[0].getField().equals(field)) {
|
||||||
|
|
||||||
|
SortField sortField = indexSort.getSort()[0];
|
||||||
|
DocIdSetIterator disi = getDocIdSetIterator(sortField, context, numericValues);
|
||||||
|
return new ConstantScoreScorer(this, score(), scoreMode, disi);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fallbackWeight.scorer(context);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isCacheable(LeafReaderContext ctx) {
|
||||||
|
// Both queries should always return the same values, so we can just check
|
||||||
|
// if the fallback query is cacheable.
|
||||||
|
return fallbackWeight.isCacheable(ctx);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Computes the document IDs that lie within the range [lowerValue, upperValue] by
|
||||||
|
* performing binary search on the field's doc values.
|
||||||
|
*
|
||||||
|
* Because doc values only allow forward iteration, we need to reload the field comparator
|
||||||
|
* every time the binary search accesses an earlier element.
|
||||||
|
*
|
||||||
|
* We must also account for missing values when performing the binary search. For this
|
||||||
|
* reason, we load the {@link FieldComparator} instead of checking the docvalues directly.
|
||||||
|
* The returned {@link DocIdSetIterator} makes sure to wrap the original docvalues to skip
|
||||||
|
* over documents with no value.
|
||||||
|
*/
|
||||||
|
private DocIdSetIterator getDocIdSetIterator(SortField sortField,
|
||||||
|
LeafReaderContext context,
|
||||||
|
DocIdSetIterator delegate) throws IOException {
|
||||||
|
long lower = sortField.getReverse() ? upperValue : lowerValue;
|
||||||
|
long upper = sortField.getReverse() ? lowerValue : upperValue;
|
||||||
|
int maxDoc = context.reader().maxDoc();
|
||||||
|
|
||||||
|
// Perform a binary search to find the first document with value >= lower.
|
||||||
|
ValueComparator comparator = loadComparator(sortField, lower, context);
|
||||||
|
int low = 0;
|
||||||
|
int high = maxDoc - 1;
|
||||||
|
|
||||||
|
while (low <= high) {
|
||||||
|
int mid = (low + high) >>> 1;
|
||||||
|
if (comparator.compare(mid) <= 0) {
|
||||||
|
high = mid - 1;
|
||||||
|
comparator = loadComparator(sortField, lower, context);
|
||||||
|
} else {
|
||||||
|
low = mid + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
int firstDocIdInclusive = high + 1;
|
||||||
|
|
||||||
|
// Perform a binary search to find the first document with value > upper.
|
||||||
|
// Since we know that upper >= lower, we can initialize the lower bound
|
||||||
|
// of the binary search to the result of the previous search.
|
||||||
|
comparator = loadComparator(sortField, upper, context);
|
||||||
|
low = firstDocIdInclusive;
|
||||||
|
high = maxDoc - 1;
|
||||||
|
|
||||||
|
while (low <= high) {
|
||||||
|
int mid = (low + high) >>> 1;
|
||||||
|
if (comparator.compare(mid) < 0) {
|
||||||
|
high = mid - 1;
|
||||||
|
comparator = loadComparator(sortField, upper, context);
|
||||||
|
} else {
|
||||||
|
low = mid + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int lastDocIdExclusive = high + 1;
|
||||||
|
return new BoundedDocSetIdIterator(firstDocIdInclusive, lastDocIdExclusive, delegate);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compares the given document's value with a stored reference value.
|
||||||
|
*/
|
||||||
|
private interface ValueComparator {
|
||||||
|
int compare(int docID) throws IOException;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ValueComparator loadComparator(SortField sortField,
|
||||||
|
long topValue,
|
||||||
|
LeafReaderContext context) throws IOException {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
FieldComparator<Long> fieldComparator = (FieldComparator<Long>) sortField.getComparator(1, 0);
|
||||||
|
fieldComparator.setTopValue(topValue);
|
||||||
|
|
||||||
|
LeafFieldComparator leafFieldComparator = fieldComparator.getLeafComparator(context);
|
||||||
|
int direction = sortField.getReverse() ? -1 : 1;
|
||||||
|
|
||||||
|
return doc -> {
|
||||||
|
int value = leafFieldComparator.compareTop(doc);
|
||||||
|
return direction * value;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A doc ID set iterator that wraps a delegate iterator and only returns doc IDs in
|
||||||
|
* the range [firstDocInclusive, lastDoc).
|
||||||
|
*/
|
||||||
|
private static class BoundedDocSetIdIterator extends DocIdSetIterator {
|
||||||
|
private final int firstDoc;
|
||||||
|
private final int lastDoc;
|
||||||
|
private final DocIdSetIterator delegate;
|
||||||
|
|
||||||
|
private int docID = -1;
|
||||||
|
|
||||||
|
BoundedDocSetIdIterator(int firstDoc,
|
||||||
|
int lastDoc,
|
||||||
|
DocIdSetIterator delegate) {
|
||||||
|
this.firstDoc = firstDoc;
|
||||||
|
this.lastDoc = lastDoc;
|
||||||
|
this.delegate = delegate;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int docID() {
|
||||||
|
return docID;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int nextDoc() throws IOException {
|
||||||
|
return advance(docID + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int advance(int target) throws IOException {
|
||||||
|
if (target < firstDoc) {
|
||||||
|
target = firstDoc;
|
||||||
|
}
|
||||||
|
|
||||||
|
int result = delegate.advance(target);
|
||||||
|
if (result < lastDoc) {
|
||||||
|
docID = result;
|
||||||
|
return docID;
|
||||||
|
} else {
|
||||||
|
return NO_MORE_DOCS;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long cost() {
|
||||||
|
return lastDoc - firstDoc;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,446 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.lucene.search;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.lucene.analysis.MockAnalyzer;
|
||||||
|
import org.apache.lucene.document.Document;
|
||||||
|
import org.apache.lucene.document.LongPoint;
|
||||||
|
import org.apache.lucene.document.SortedNumericDocValuesField;
|
||||||
|
import org.apache.lucene.document.SortedSetDocValuesField;
|
||||||
|
import org.apache.lucene.index.DirectoryReader;
|
||||||
|
import org.apache.lucene.index.IndexReader;
|
||||||
|
import org.apache.lucene.index.IndexWriterConfig;
|
||||||
|
import org.apache.lucene.index.LeafReaderContext;
|
||||||
|
import org.apache.lucene.index.RandomIndexWriter;
|
||||||
|
import org.apache.lucene.store.Directory;
|
||||||
|
import org.apache.lucene.util.BytesRef;
|
||||||
|
import org.apache.lucene.util.LuceneTestCase;
|
||||||
|
import org.apache.lucene.util.TestUtil;
|
||||||
|
|
||||||
|
import static org.hamcrest.CoreMatchers.instanceOf;
|
||||||
|
|
||||||
|
public class TestIndexSortSortedNumericDocValuesRangeQuery extends LuceneTestCase {
|
||||||
|
|
||||||
|
public void testSameHitsAsPointRangeQuery() throws IOException {
|
||||||
|
final int iters = atLeast(10);
|
||||||
|
for (int iter = 0; iter < iters; ++iter) {
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
|
||||||
|
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
|
||||||
|
boolean reverse = random().nextBoolean();
|
||||||
|
SortField sortField = new SortedNumericSortField("dv", SortField.Type.LONG, reverse);
|
||||||
|
sortField.setMissingValue(random().nextLong());
|
||||||
|
iwc.setIndexSort(new Sort(sortField));
|
||||||
|
|
||||||
|
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
|
||||||
|
|
||||||
|
final int numDocs = atLeast(100);
|
||||||
|
for (int i = 0; i < numDocs; ++i) {
|
||||||
|
Document doc = new Document();
|
||||||
|
final int numValues = TestUtil.nextInt(random(), 0, 1);
|
||||||
|
for (int j = 0; j < numValues; ++j) {
|
||||||
|
final long value = TestUtil.nextLong(random(), -100, 10000);
|
||||||
|
doc.add(new SortedNumericDocValuesField("dv", value));
|
||||||
|
doc.add(new LongPoint("idx", value));
|
||||||
|
}
|
||||||
|
iw.addDocument(doc);
|
||||||
|
}
|
||||||
|
if (random().nextBoolean()) {
|
||||||
|
iw.deleteDocuments(LongPoint.newRangeQuery("idx", 0L, 10L));
|
||||||
|
}
|
||||||
|
final IndexReader reader = iw.getReader();
|
||||||
|
final IndexSearcher searcher = newSearcher(reader, false);
|
||||||
|
iw.close();
|
||||||
|
|
||||||
|
for (int i = 0; i < 100; ++i) {
|
||||||
|
final long min = random().nextBoolean() ? Long.MIN_VALUE : TestUtil.nextLong(random(), -100, 10000);
|
||||||
|
final long max = random().nextBoolean() ? Long.MAX_VALUE : TestUtil.nextLong(random(), -100, 10000);
|
||||||
|
final Query q1 = LongPoint.newRangeQuery("idx", min, max);
|
||||||
|
final Query q2 = createQuery("dv", min, max);
|
||||||
|
assertSameHits(searcher, q1, q2, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
reader.close();
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertSameHits(IndexSearcher searcher, Query q1, Query q2, boolean scores) throws IOException {
|
||||||
|
final int maxDoc = searcher.getIndexReader().maxDoc();
|
||||||
|
final TopDocs td1 = searcher.search(q1, maxDoc, scores ? Sort.RELEVANCE : Sort.INDEXORDER);
|
||||||
|
final TopDocs td2 = searcher.search(q2, maxDoc, scores ? Sort.RELEVANCE : Sort.INDEXORDER);
|
||||||
|
assertEquals(td1.totalHits.value, td2.totalHits.value);
|
||||||
|
for (int i = 0; i < td1.scoreDocs.length; ++i) {
|
||||||
|
assertEquals(td1.scoreDocs[i].doc, td2.scoreDocs[i].doc);
|
||||||
|
if (scores) {
|
||||||
|
assertEquals(td1.scoreDocs[i].score, td2.scoreDocs[i].score, 10e-7);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testEquals() {
|
||||||
|
Query q1 = createQuery("foo", 3, 5);
|
||||||
|
QueryUtils.checkEqual(q1, createQuery("foo", 3, 5));
|
||||||
|
QueryUtils.checkUnequal(q1, createQuery("foo", 3, 6));
|
||||||
|
QueryUtils.checkUnequal(q1, createQuery("foo", 4, 5));
|
||||||
|
QueryUtils.checkUnequal(q1, createQuery("bar", 3, 5));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testToString() {
|
||||||
|
Query q1 = createQuery("foo", 3, 5);
|
||||||
|
assertEquals("foo:[3 TO 5]", q1.toString());
|
||||||
|
assertEquals("[3 TO 5]", q1.toString("foo"));
|
||||||
|
assertEquals("foo:[3 TO 5]", q1.toString("bar"));
|
||||||
|
|
||||||
|
Query q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), true, true);
|
||||||
|
assertEquals("foo:[[62 61 72] TO [62 61 7a]]", q2.toString());
|
||||||
|
q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), false, true);
|
||||||
|
assertEquals("foo:{[62 61 72] TO [62 61 7a]]", q2.toString());
|
||||||
|
q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), false, false);
|
||||||
|
assertEquals("foo:{[62 61 72] TO [62 61 7a]}", q2.toString());
|
||||||
|
q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), null, true, true);
|
||||||
|
assertEquals("foo:[[62 61 72] TO *}", q2.toString());
|
||||||
|
q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", null, new BytesRef("baz"), true, true);
|
||||||
|
assertEquals("foo:{* TO [62 61 7a]]", q2.toString());
|
||||||
|
assertEquals("{* TO [62 61 7a]]", q2.toString("foo"));
|
||||||
|
assertEquals("foo:{* TO [62 61 7a]]", q2.toString("bar"));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testIndexSortDocValuesWithEvenLength() throws Exception {
|
||||||
|
testIndexSortDocValuesWithEvenLength(false);
|
||||||
|
testIndexSortDocValuesWithEvenLength(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testIndexSortDocValuesWithEvenLength(boolean reverse) throws Exception {
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
|
||||||
|
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
|
||||||
|
Sort indexSort = new Sort(new SortedNumericSortField("field", SortField.Type.LONG, reverse));
|
||||||
|
iwc.setIndexSort(indexSort);
|
||||||
|
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
|
||||||
|
|
||||||
|
writer.addDocument(createDocument("field", -80));
|
||||||
|
writer.addDocument(createDocument("field", -5));
|
||||||
|
writer.addDocument(createDocument("field", 0));
|
||||||
|
writer.addDocument(createDocument("field", 0));
|
||||||
|
writer.addDocument(createDocument("field", 30));
|
||||||
|
writer.addDocument(createDocument("field", 35));
|
||||||
|
|
||||||
|
DirectoryReader reader = writer.getReader();
|
||||||
|
IndexSearcher searcher = newSearcher(reader);
|
||||||
|
|
||||||
|
// Test ranges consisting of one value.
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", -80, -80)));
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", -5, -5)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", 0, 0)));
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", 30, 30)));
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", 35, 35)));
|
||||||
|
|
||||||
|
assertEquals(0, searcher.count(createQuery("field", -90, -90)));
|
||||||
|
assertEquals(0, searcher.count(createQuery("field", 5, 5)));
|
||||||
|
assertEquals(0, searcher.count(createQuery("field", 40, 40)));
|
||||||
|
|
||||||
|
// Test the lower end of the document value range.
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -90, -4)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -80, -4)));
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", -70, -4)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -80, -5)));
|
||||||
|
|
||||||
|
// Test the upper end of the document value range.
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", 25, 34)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", 25, 35)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", 25, 36)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", 30, 35)));
|
||||||
|
|
||||||
|
// Test multiple occurrences of the same value.
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -4, 4)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -4, 0)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", 0, 4)));
|
||||||
|
assertEquals(3, searcher.count(createQuery("field", 0, 30)));
|
||||||
|
|
||||||
|
// Test ranges that span all documents.
|
||||||
|
assertEquals(6, searcher.count(createQuery("field", -80, 35)));
|
||||||
|
assertEquals(6, searcher.count(createQuery("field", -90, 40)));
|
||||||
|
|
||||||
|
writer.close();
|
||||||
|
reader.close();
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testIndexSortDocValuesWithOddLength() throws Exception {
|
||||||
|
testIndexSortDocValuesWithOddLength(false);
|
||||||
|
testIndexSortDocValuesWithOddLength(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testIndexSortDocValuesWithOddLength(boolean reverse) throws Exception {
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
|
||||||
|
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
|
||||||
|
Sort indexSort = new Sort(new SortedNumericSortField("field", SortField.Type.LONG, reverse));
|
||||||
|
iwc.setIndexSort(indexSort);
|
||||||
|
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
|
||||||
|
|
||||||
|
writer.addDocument(createDocument("field", -80));
|
||||||
|
writer.addDocument(createDocument("field", -5));
|
||||||
|
writer.addDocument(createDocument("field", 0));
|
||||||
|
writer.addDocument(createDocument("field", 0));
|
||||||
|
writer.addDocument(createDocument("field", 5));
|
||||||
|
writer.addDocument(createDocument("field", 30));
|
||||||
|
writer.addDocument(createDocument("field", 35));
|
||||||
|
|
||||||
|
DirectoryReader reader = writer.getReader();
|
||||||
|
IndexSearcher searcher = newSearcher(reader);
|
||||||
|
|
||||||
|
// Test ranges consisting of one value.
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", -80, -80)));
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", -5, -5)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", 0, 0)));
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", 5, 5)));
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", 30, 30)));
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", 35, 35)));
|
||||||
|
|
||||||
|
assertEquals(0, searcher.count(createQuery("field", -90, -90)));
|
||||||
|
assertEquals(0, searcher.count(createQuery("field", 6, 6)));
|
||||||
|
assertEquals(0, searcher.count(createQuery("field", 40, 40)));
|
||||||
|
|
||||||
|
// Test the lower end of the document value range.
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -90, -4)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -80, -4)));
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", -70, -4)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -80, -5)));
|
||||||
|
|
||||||
|
// Test the upper end of the document value range.
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", 25, 34)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", 25, 35)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", 25, 36)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", 30, 35)));
|
||||||
|
|
||||||
|
// Test multiple occurrences of the same value.
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -4, 4)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -4, 0)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", 0, 4)));
|
||||||
|
assertEquals(4, searcher.count(createQuery("field", 0, 30)));
|
||||||
|
|
||||||
|
// Test ranges that span all documents.
|
||||||
|
assertEquals(7, searcher.count(createQuery("field", -80, 35)));
|
||||||
|
assertEquals(7, searcher.count(createQuery("field", -90, 40)));
|
||||||
|
|
||||||
|
writer.close();
|
||||||
|
reader.close();
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testIndexSortDocValuesWithSingleValue() throws Exception {
|
||||||
|
testIndexSortDocValuesWithSingleValue(false);
|
||||||
|
testIndexSortDocValuesWithSingleValue(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void testIndexSortDocValuesWithSingleValue(boolean reverse) throws IOException{
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
|
||||||
|
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
|
||||||
|
Sort indexSort = new Sort(new SortedNumericSortField("field", SortField.Type.LONG, reverse));
|
||||||
|
iwc.setIndexSort(indexSort);
|
||||||
|
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
|
||||||
|
|
||||||
|
writer.addDocument(createDocument("field", 42));
|
||||||
|
|
||||||
|
DirectoryReader reader = writer.getReader();
|
||||||
|
IndexSearcher searcher = newSearcher(reader);
|
||||||
|
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", 42, 43)));
|
||||||
|
assertEquals(1, searcher.count(createQuery("field", 42, 42)));
|
||||||
|
assertEquals(0, searcher.count(createQuery("field", 41, 41)));
|
||||||
|
assertEquals(0, searcher.count(createQuery("field", 43, 43)));
|
||||||
|
|
||||||
|
writer.close();
|
||||||
|
reader.close();
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testIndexSortMissingValues() throws Exception {
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
|
||||||
|
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
|
||||||
|
SortField sortField = new SortedNumericSortField("field", SortField.Type.LONG);
|
||||||
|
sortField.setMissingValue(random().nextLong());
|
||||||
|
iwc.setIndexSort(new Sort(sortField));
|
||||||
|
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
|
||||||
|
|
||||||
|
writer.addDocument(createDocument("field", -80));
|
||||||
|
writer.addDocument(createDocument("field", -5));
|
||||||
|
writer.addDocument(createDocument("field", 0));
|
||||||
|
writer.addDocument(createDocument("field", 35));
|
||||||
|
|
||||||
|
writer.addDocument(createDocument("other-field", 0));
|
||||||
|
writer.addDocument(createDocument("other-field", 10));
|
||||||
|
writer.addDocument(createDocument("other-field", 20));
|
||||||
|
|
||||||
|
DirectoryReader reader = writer.getReader();
|
||||||
|
IndexSearcher searcher = newSearcher(reader);
|
||||||
|
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -70, 0)));
|
||||||
|
assertEquals(2, searcher.count(createQuery("field", -2, 35)));
|
||||||
|
|
||||||
|
assertEquals(4, searcher.count(createQuery("field", -80, 35)));
|
||||||
|
assertEquals(4, searcher.count(createQuery("field", Long.MIN_VALUE, Long.MAX_VALUE)));
|
||||||
|
|
||||||
|
writer.close();
|
||||||
|
reader.close();
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testNoDocuments() throws IOException {
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||||
|
writer.addDocument(new Document());
|
||||||
|
IndexReader reader = writer.getReader();
|
||||||
|
IndexSearcher searcher = newSearcher(reader);
|
||||||
|
Query query = createQuery("foo", 2, 4);
|
||||||
|
Weight w = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE, 1);
|
||||||
|
assertNull(w.scorer(searcher.getIndexReader().leaves().get(0)));
|
||||||
|
|
||||||
|
writer.close();
|
||||||
|
reader.close();
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testRewriteExhaustiveRange() throws IOException {
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||||
|
writer.addDocument(new Document());
|
||||||
|
IndexReader reader = writer.getReader();
|
||||||
|
|
||||||
|
Query query = createQuery("field", Long.MIN_VALUE, Long.MAX_VALUE);
|
||||||
|
Query rewrittenQuery = query.rewrite(reader);
|
||||||
|
assertEquals(new DocValuesFieldExistsQuery("field"), rewrittenQuery);
|
||||||
|
|
||||||
|
writer.close();
|
||||||
|
reader.close();
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testRewriteFallbackQuery() throws IOException {
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||||
|
writer.addDocument(new Document());
|
||||||
|
IndexReader reader = writer.getReader();
|
||||||
|
|
||||||
|
// Create an (unrealistic) fallback query that is sure to be rewritten.
|
||||||
|
Query fallbackQuery = new BooleanQuery.Builder().build();
|
||||||
|
Query query = new IndexSortSortedNumericDocValuesRangeQuery("field", 1, 42, fallbackQuery);
|
||||||
|
|
||||||
|
Query rewrittenQuery = query.rewrite(reader);
|
||||||
|
assertNotEquals(query, rewrittenQuery);
|
||||||
|
assertThat(rewrittenQuery, instanceOf(IndexSortSortedNumericDocValuesRangeQuery.class));
|
||||||
|
|
||||||
|
IndexSortSortedNumericDocValuesRangeQuery rangeQuery = (IndexSortSortedNumericDocValuesRangeQuery) rewrittenQuery;
|
||||||
|
assertEquals(new MatchNoDocsQuery(), rangeQuery.getFallbackQuery());
|
||||||
|
|
||||||
|
writer.close();
|
||||||
|
reader.close();
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test that the index sort optimization not activated if there is no index sort.
|
||||||
|
*/
|
||||||
|
public void testNoIndexSort() throws Exception {
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
|
||||||
|
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||||
|
writer.addDocument(createDocument("field", 0));
|
||||||
|
|
||||||
|
testIndexSortOptimizationDeactivated(writer);
|
||||||
|
|
||||||
|
writer.close();
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test that the index sort optimization is not activated when the sort is
|
||||||
|
* on the wrong field.
|
||||||
|
*/
|
||||||
|
public void testIndexSortOnWrongField() throws Exception {
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
|
||||||
|
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
|
||||||
|
Sort indexSort = new Sort(new SortedNumericSortField("other-field", SortField.Type.LONG));
|
||||||
|
iwc.setIndexSort(indexSort);
|
||||||
|
|
||||||
|
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
|
||||||
|
writer.addDocument(createDocument("field", 0));
|
||||||
|
|
||||||
|
testIndexSortOptimizationDeactivated(writer);
|
||||||
|
|
||||||
|
writer.close();
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test that the index sort optimization is not activated when some documents
|
||||||
|
* have multiple values.
|
||||||
|
*/
|
||||||
|
public void testMultiDocValues() throws Exception {
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
|
||||||
|
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
|
||||||
|
Sort indexSort = new Sort(new SortedNumericSortField("field", SortField.Type.LONG));
|
||||||
|
iwc.setIndexSort(indexSort);
|
||||||
|
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
|
||||||
|
|
||||||
|
Document doc = new Document();
|
||||||
|
doc.add(new SortedNumericDocValuesField("field", 0));
|
||||||
|
doc.add(new SortedNumericDocValuesField("field", 10));
|
||||||
|
writer.addDocument(doc);
|
||||||
|
|
||||||
|
testIndexSortOptimizationDeactivated(writer);
|
||||||
|
|
||||||
|
writer.close();
|
||||||
|
dir.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testIndexSortOptimizationDeactivated(RandomIndexWriter writer) throws IOException {
|
||||||
|
DirectoryReader reader = writer.getReader();
|
||||||
|
IndexSearcher searcher = newSearcher(reader);
|
||||||
|
|
||||||
|
Query query = createQuery("field", 0, 0);
|
||||||
|
Weight weight = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0F);
|
||||||
|
|
||||||
|
// Check that the two-phase iterator is not null, indicating that we've fallen
|
||||||
|
// back to SortedNumericDocValuesField.newSlowRangeQuery.
|
||||||
|
for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
|
||||||
|
Scorer scorer = weight.scorer(context);
|
||||||
|
assertNotNull(scorer.twoPhaseIterator());
|
||||||
|
}
|
||||||
|
|
||||||
|
reader.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private Document createDocument(String field, long value) {
|
||||||
|
Document doc = new Document();
|
||||||
|
doc.add(new SortedNumericDocValuesField(field, value));
|
||||||
|
return doc;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Query createQuery(String field, long lowerValue, long upperValue) {
|
||||||
|
Query fallbackQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, lowerValue, upperValue);
|
||||||
|
return new IndexSortSortedNumericDocValuesRangeQuery(field, lowerValue, upperValue, fallbackQuery);
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue