diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 1e486bc1fda..ae2f9f98228 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -131,6 +131,9 @@ API Changes positions are not available. (Ryan Ernst) +* LUCENE-6268: Replace FieldValueFilter and DocValuesRangeFilter with equivalent + queries that support approximations. (Adrien Grand) + Other * LUCENE-6248: Remove unused odd constants from StandardSyntaxParser.jj diff --git a/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationDocValuesField.java b/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationDocValuesField.java index 3910e742156..ef3423d4c24 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationDocValuesField.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationDocValuesField.java @@ -21,7 +21,7 @@ import java.text.Collator; import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedDocValuesField; -import org.apache.lucene.search.DocValuesRangeFilter; +import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.util.BytesRef; /** @@ -29,7 +29,7 @@ import org.apache.lucene.util.BytesRef; *

* This is more efficient that {@link CollationKeyAnalyzer} if the field * only has one value: no uninversion is necessary to sort on the field, - * locale-sensitive range queries can still work via {@link DocValuesRangeFilter}, + * locale-sensitive range queries can still work via {@link DocValuesRangeQuery}, * and the underlying data structures built at index-time are likely more efficient * and use less memory than FieldCache. */ diff --git a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java index 8334065a92f..c0167958314 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java @@ -25,9 +25,10 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.DocValuesRangeFilter; +import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; @@ -36,7 +37,6 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; @@ -111,7 +111,7 @@ public class TestCollationDocValuesField extends LuceneTestCase { String end = TestUtil.randomSimpleString(random()); BytesRef lowerVal = new BytesRef(collator.getCollationKey(start).toByteArray()); BytesRef upperVal = new BytesRef(collator.getCollationKey(end).toByteArray()); - Query query = new ConstantScoreQuery(DocValuesRangeFilter.newBytesRefRange("collated", lowerVal, upperVal, true, true)); + Query query = DocValuesRangeQuery.newBytesRefRange("collated", lowerVal, upperVal, true, true); doTestRanges(is, start, end, query, collator); } } finally { diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationDocValuesField.java b/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationDocValuesField.java index b70b0aee352..0ef292ee031 100644 --- a/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationDocValuesField.java +++ b/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationDocValuesField.java @@ -19,7 +19,7 @@ package org.apache.lucene.collation; import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedDocValuesField; -import org.apache.lucene.search.DocValuesRangeFilter; +import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.util.BytesRef; import com.ibm.icu.text.Collator; @@ -30,7 +30,7 @@ import com.ibm.icu.text.RawCollationKey; *

* This is more efficient that {@link ICUCollationKeyAnalyzer} if the field * only has one value: no uninversion is necessary to sort on the field, - * locale-sensitive range queries can still work via {@link DocValuesRangeFilter}, + * locale-sensitive range queries can still work via {@link DocValuesRangeQuery}, * and the underlying data structures built at index-time are likely more efficient * and use less memory than FieldCache. */ diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationDocValuesField.java b/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationDocValuesField.java index dff883bbf44..54af2e95851 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationDocValuesField.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationDocValuesField.java @@ -22,9 +22,9 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.DocValuesRangeFilter; +import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; @@ -33,7 +33,6 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; @@ -109,7 +108,7 @@ public class TestICUCollationDocValuesField extends LuceneTestCase { String end = TestUtil.randomSimpleString(random()); BytesRef lowerVal = new BytesRef(collator.getCollationKey(start).toByteArray()); BytesRef upperVal = new BytesRef(collator.getCollationKey(end).toByteArray()); - Query query = new ConstantScoreQuery(DocValuesRangeFilter.newBytesRefRange("collated", lowerVal, upperVal, true, true)); + Query query = DocValuesRangeQuery.newBytesRefRange("collated", lowerVal, upperVal, true, true); doTestRanges(is, start, end, query, collator); } diff --git a/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java b/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java index 55df0db6a37..fe26e52382a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java @@ -58,7 +58,39 @@ public abstract class DocIdSetIterator { } }; } - + + /** A {@link DocIdSetIterator} that matches all documents up to + * {@code maxDoc - 1}. */ + public static final DocIdSetIterator all(int maxDoc) { + return new DocIdSetIterator() { + int doc = -1; + + @Override + public int docID() { + return doc; + } + + @Override + public int nextDoc() throws IOException { + return advance(doc + 1); + } + + @Override + public int advance(int target) throws IOException { + doc = target; + if (doc >= maxDoc) { + doc = NO_MORE_DOCS; + } + return doc; + } + + @Override + public long cost() { + return maxDoc; + } + }; + } + /** * When returned by {@link #nextDoc()}, {@link #advance(int)} and * {@link #docID()} it means there are no more docs in the iterator. diff --git a/lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRangeFilter.java b/lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRangeFilter.java deleted file mode 100644 index 4a137f858c3..00000000000 --- a/lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRangeFilter.java +++ /dev/null @@ -1,168 +0,0 @@ -package org.apache.lucene.search; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; - -/** - * A range filter built on top of a cached multi-valued term field (from {@link org.apache.lucene.index.LeafReader#getSortedSetDocValues}). - * - *

Like {@link DocValuesRangeFilter}, this is just a specialized range query versus - * using a TermRangeQuery with {@link DocTermOrdsRewriteMethod}: it will only do - * two ordinal to term lookups.

- */ - -public abstract class DocTermOrdsRangeFilter extends Filter { - final String field; - final BytesRef lowerVal; - final BytesRef upperVal; - final boolean includeLower; - final boolean includeUpper; - - private DocTermOrdsRangeFilter(String field, BytesRef lowerVal, BytesRef upperVal, boolean includeLower, boolean includeUpper) { - this.field = field; - this.lowerVal = lowerVal; - this.upperVal = upperVal; - this.includeLower = includeLower; - this.includeUpper = includeUpper; - } - - /** This method is implemented for each data type */ - @Override - public abstract DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException; - - /** - * Creates a BytesRef range filter using {@link org.apache.lucene.index.LeafReader#getSortedSetDocValues}. This works with all - * fields containing zero or one term in the field. The range can be half-open by setting one - * of the values to null. - */ - public static DocTermOrdsRangeFilter newBytesRefRange(String field, BytesRef lowerVal, BytesRef upperVal, boolean includeLower, boolean includeUpper) { - return new DocTermOrdsRangeFilter(field, lowerVal, upperVal, includeLower, includeUpper) { - @Override - public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException { - final SortedSetDocValues docTermOrds = DocValues.getSortedSet(context.reader(), field); - final long lowerPoint = lowerVal == null ? -1 : docTermOrds.lookupTerm(lowerVal); - final long upperPoint = upperVal == null ? -1 : docTermOrds.lookupTerm(upperVal); - - final long inclusiveLowerPoint, inclusiveUpperPoint; - - // Hints: - // * binarySearchLookup returns -1, if value was null. - // * the value is <0 if no exact hit was found, the returned value - // is (-(insertion point) - 1) - if (lowerPoint == -1 && lowerVal == null) { - inclusiveLowerPoint = 0; - } else if (includeLower && lowerPoint >= 0) { - inclusiveLowerPoint = lowerPoint; - } else if (lowerPoint >= 0) { - inclusiveLowerPoint = lowerPoint + 1; - } else { - inclusiveLowerPoint = Math.max(0, -lowerPoint - 1); - } - - if (upperPoint == -1 && upperVal == null) { - inclusiveUpperPoint = Long.MAX_VALUE; - } else if (includeUpper && upperPoint >= 0) { - inclusiveUpperPoint = upperPoint; - } else if (upperPoint >= 0) { - inclusiveUpperPoint = upperPoint - 1; - } else { - inclusiveUpperPoint = -upperPoint - 2; - } - - if (inclusiveUpperPoint < 0 || inclusiveLowerPoint > inclusiveUpperPoint) { - return null; - } - - assert inclusiveLowerPoint >= 0 && inclusiveUpperPoint >= 0; - - return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) { - @Override - protected final boolean matchDoc(int doc) { - docTermOrds.setDocument(doc); - long ord; - while ((ord = docTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) { - if (ord > inclusiveUpperPoint) { - return false; - } else if (ord >= inclusiveLowerPoint) { - return true; - } - } - return false; - } - }; - } - }; - } - - @Override - public final String toString(String defaultField) { - final StringBuilder sb = new StringBuilder(field).append(":"); - return sb.append(includeLower ? '[' : '{') - .append((lowerVal == null) ? "*" : lowerVal.toString()) - .append(" TO ") - .append((upperVal == null) ? "*" : upperVal.toString()) - .append(includeUpper ? ']' : '}') - .toString(); - } - - @Override - public final boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof DocTermOrdsRangeFilter)) return false; - DocTermOrdsRangeFilter other = (DocTermOrdsRangeFilter) o; - - if (!this.field.equals(other.field) - || this.includeLower != other.includeLower - || this.includeUpper != other.includeUpper - ) { return false; } - if (this.lowerVal != null ? !this.lowerVal.equals(other.lowerVal) : other.lowerVal != null) return false; - if (this.upperVal != null ? !this.upperVal.equals(other.upperVal) : other.upperVal != null) return false; - return true; - } - - @Override - public final int hashCode() { - int h = field.hashCode(); - h ^= (lowerVal != null) ? lowerVal.hashCode() : 550356204; - h = (h << 1) | (h >>> 31); // rotate to distinguish lower from upper - h ^= (upperVal != null) ? upperVal.hashCode() : -1674416163; - h ^= (includeLower ? 1549299360 : -365038026) ^ (includeUpper ? 1721088258 : 1948649653); - return h; - } - - /** Returns the field name for this filter */ - public String getField() { return field; } - - /** Returns true if the lower endpoint is inclusive */ - public boolean includesLower() { return includeLower; } - - /** Returns true if the upper endpoint is inclusive */ - public boolean includesUpper() { return includeUpper; } - - /** Returns the lower value of this range filter */ - public BytesRef getLowerVal() { return lowerVal; } - - /** Returns the upper value of this range filter */ - public BytesRef getUpperVal() { return upperVal; } -} diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeFilter.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeFilter.java deleted file mode 100644 index dd6778db9fc..00000000000 --- a/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeFilter.java +++ /dev/null @@ -1,426 +0,0 @@ -package org.apache.lucene.search; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.document.DoubleField; // for javadocs -import org.apache.lucene.document.FloatField; // for javadocs -import org.apache.lucene.document.IntField; // for javadocs -import org.apache.lucene.document.LongField; // for javadocs -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.NumericUtils; - -/** - * A range filter built on top of numeric doc values field - * (from {@link org.apache.lucene.index.LeafReader#getNumericDocValues(String)}). - * - *

{@code DocValuesRangeFilter} builds a single cache for the field the first time it is used. - * Each subsequent {@code DocValuesRangeFilter} on the same field then reuses this cache, - * even if the range itself changes. - * - *

This means that {@code DocValuesRangeFilter} is much faster (sometimes more than 100x as fast) - * as building a {@link TermRangeFilter}, if using a {@link #newStringRange}. - * However, if the range never changes it is slower (around 2x as slow) than building - * a CachingWrapperFilter on top of a single {@link TermRangeFilter}. - * - * For numeric data types, this filter may be significantly faster than {@link NumericRangeFilter}. - * Furthermore, it does not need the numeric values encoded - * by {@link IntField}, {@link FloatField}, {@link - * LongField} or {@link DoubleField}. But - * it has the problem that it only works with exact one value/document (see below). - * - *

As with all {@link org.apache.lucene.index.LeafReader#getNumericDocValues} based functionality, - * {@code DocValuesRangeFilter} is only valid for - * fields which exact one term for each document (except for {@link #newStringRange} - * where 0 terms are also allowed). Due to historical reasons, for numeric ranges - * all terms that do not have a numeric value, 0 is assumed. - * - *

Thus it works on dates, prices and other single value fields but will not work on - * regular text fields. It is preferable to use a NOT_ANALYZED field to ensure that - * there is only a single term. - * - *

This class does not have an constructor, use one of the static factory methods available, - * that create a correct instance for different data types. - */ -// TODO: use docsWithField to handle empty properly -public abstract class DocValuesRangeFilter extends Filter { - final String field; - final T lowerVal; - final T upperVal; - final boolean includeLower; - final boolean includeUpper; - - private DocValuesRangeFilter(String field, T lowerVal, T upperVal, boolean includeLower, boolean includeUpper) { - this.field = field; - this.lowerVal = lowerVal; - this.upperVal = upperVal; - this.includeLower = includeLower; - this.includeUpper = includeUpper; - } - - /** This method is implemented for each data type */ - @Override - public abstract DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException; - - /** - * Creates a string range filter using {@link org.apache.lucene.index.LeafReader#getSortedDocValues(String)}. This works with all - * fields containing zero or one term in the field. The range can be half-open by setting one - * of the values to null. - */ - public static DocValuesRangeFilter newStringRange(String field, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) { - return new DocValuesRangeFilter(field, lowerVal, upperVal, includeLower, includeUpper) { - @Override - public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException { - final SortedDocValues fcsi = DocValues.getSorted(context.reader(), field); - final int lowerPoint = lowerVal == null ? -1 : fcsi.lookupTerm(new BytesRef(lowerVal)); - final int upperPoint = upperVal == null ? -1 : fcsi.lookupTerm(new BytesRef(upperVal)); - - final int inclusiveLowerPoint, inclusiveUpperPoint; - - // Hints: - // * binarySearchLookup returns -1, if value was null. - // * the value is <0 if no exact hit was found, the returned value - // is (-(insertion point) - 1) - if (lowerPoint == -1 && lowerVal == null) { - inclusiveLowerPoint = 0; - } else if (includeLower && lowerPoint >= 0) { - inclusiveLowerPoint = lowerPoint; - } else if (lowerPoint >= 0) { - inclusiveLowerPoint = lowerPoint + 1; - } else { - inclusiveLowerPoint = Math.max(0, -lowerPoint - 1); - } - - if (upperPoint == -1 && upperVal == null) { - inclusiveUpperPoint = Integer.MAX_VALUE; - } else if (includeUpper && upperPoint >= 0) { - inclusiveUpperPoint = upperPoint; - } else if (upperPoint >= 0) { - inclusiveUpperPoint = upperPoint - 1; - } else { - inclusiveUpperPoint = -upperPoint - 2; - } - - if (inclusiveUpperPoint < 0 || inclusiveLowerPoint > inclusiveUpperPoint) { - return null; - } - - assert inclusiveLowerPoint >= 0 && inclusiveUpperPoint >= 0; - - return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) { - @Override - protected final boolean matchDoc(int doc) { - final int docOrd = fcsi.getOrd(doc); - return docOrd >= inclusiveLowerPoint && docOrd <= inclusiveUpperPoint; - } - }; - } - }; - } - - /** - * Creates a BytesRef range filter using {@link org.apache.lucene.index.LeafReader#getSortedDocValues(String)}. This works with all - * fields containing zero or one term in the field. The range can be half-open by setting one - * of the values to null. - */ - // TODO: bogus that newStringRange doesnt share this code... generics hell - public static DocValuesRangeFilter newBytesRefRange(String field, BytesRef lowerVal, BytesRef upperVal, boolean includeLower, boolean includeUpper) { - return new DocValuesRangeFilter(field, lowerVal, upperVal, includeLower, includeUpper) { - @Override - public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException { - final SortedDocValues fcsi = DocValues.getSorted(context.reader(), field); - final int lowerPoint = lowerVal == null ? -1 : fcsi.lookupTerm(lowerVal); - final int upperPoint = upperVal == null ? -1 : fcsi.lookupTerm(upperVal); - - final int inclusiveLowerPoint, inclusiveUpperPoint; - - // Hints: - // * binarySearchLookup returns -1, if value was null. - // * the value is <0 if no exact hit was found, the returned value - // is (-(insertion point) - 1) - if (lowerPoint == -1 && lowerVal == null) { - inclusiveLowerPoint = 0; - } else if (includeLower && lowerPoint >= 0) { - inclusiveLowerPoint = lowerPoint; - } else if (lowerPoint >= 0) { - inclusiveLowerPoint = lowerPoint + 1; - } else { - inclusiveLowerPoint = Math.max(0, -lowerPoint - 1); - } - - if (upperPoint == -1 && upperVal == null) { - inclusiveUpperPoint = Integer.MAX_VALUE; - } else if (includeUpper && upperPoint >= 0) { - inclusiveUpperPoint = upperPoint; - } else if (upperPoint >= 0) { - inclusiveUpperPoint = upperPoint - 1; - } else { - inclusiveUpperPoint = -upperPoint - 2; - } - - if (inclusiveUpperPoint < 0 || inclusiveLowerPoint > inclusiveUpperPoint) { - return null; - } - - assert inclusiveLowerPoint >= 0 && inclusiveUpperPoint >= 0; - - return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) { - @Override - protected final boolean matchDoc(int doc) { - final int docOrd = fcsi.getOrd(doc); - return docOrd >= inclusiveLowerPoint && docOrd <= inclusiveUpperPoint; - } - }; - } - }; - } - - /** - * Creates a numeric range filter using {@link org.apache.lucene.index.LeafReader#getSortedDocValues(String)}. This works with all - * int fields containing exactly one numeric term in the field. The range can be half-open by setting one - * of the values to null. - */ - public static DocValuesRangeFilter newIntRange(String field, Integer lowerVal, Integer upperVal, boolean includeLower, boolean includeUpper) { - return new DocValuesRangeFilter(field, lowerVal, upperVal, includeLower, includeUpper) { - @Override - public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException { - final int inclusiveLowerPoint, inclusiveUpperPoint; - if (lowerVal != null) { - int i = lowerVal.intValue(); - if (!includeLower && i == Integer.MAX_VALUE) - return null; - inclusiveLowerPoint = includeLower ? i : (i + 1); - } else { - inclusiveLowerPoint = Integer.MIN_VALUE; - } - if (upperVal != null) { - int i = upperVal.intValue(); - if (!includeUpper && i == Integer.MIN_VALUE) - return null; - inclusiveUpperPoint = includeUpper ? i : (i - 1); - } else { - inclusiveUpperPoint = Integer.MAX_VALUE; - } - - if (inclusiveLowerPoint > inclusiveUpperPoint) - return null; - - final NumericDocValues values = DocValues.getNumeric(context.reader(), field); - return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) { - @Override - protected boolean matchDoc(int doc) { - final int value = (int) values.get(doc); - return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint; - } - }; - } - }; - } - - /** - * Creates a numeric range filter using {@link org.apache.lucene.index.LeafReader#getNumericDocValues(String)}. This works with all - * long fields containing exactly one numeric term in the field. The range can be half-open by setting one - * of the values to null. - */ - public static DocValuesRangeFilter newLongRange(String field, Long lowerVal, Long upperVal, boolean includeLower, boolean includeUpper) { - return new DocValuesRangeFilter(field, lowerVal, upperVal, includeLower, includeUpper) { - @Override - public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException { - final long inclusiveLowerPoint, inclusiveUpperPoint; - if (lowerVal != null) { - long i = lowerVal.longValue(); - if (!includeLower && i == Long.MAX_VALUE) - return null; - inclusiveLowerPoint = includeLower ? i : (i + 1L); - } else { - inclusiveLowerPoint = Long.MIN_VALUE; - } - if (upperVal != null) { - long i = upperVal.longValue(); - if (!includeUpper && i == Long.MIN_VALUE) - return null; - inclusiveUpperPoint = includeUpper ? i : (i - 1L); - } else { - inclusiveUpperPoint = Long.MAX_VALUE; - } - - if (inclusiveLowerPoint > inclusiveUpperPoint) - return null; - - final NumericDocValues values = DocValues.getNumeric(context.reader(), field); - return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) { - @Override - protected boolean matchDoc(int doc) { - final long value = values.get(doc); - return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint; - } - }; - } - }; - } - - /** - * Creates a numeric range filter using {@link org.apache.lucene.index.LeafReader#getNumericDocValues(String)}. This works with all - * float fields containing exactly one numeric term in the field. The range can be half-open by setting one - * of the values to null. - */ - public static DocValuesRangeFilter newFloatRange(String field, Float lowerVal, Float upperVal, boolean includeLower, boolean includeUpper) { - return new DocValuesRangeFilter(field, lowerVal, upperVal, includeLower, includeUpper) { - @Override - public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException { - // we transform the floating point numbers to sortable integers - // using NumericUtils to easier find the next bigger/lower value - final float inclusiveLowerPoint, inclusiveUpperPoint; - if (lowerVal != null) { - float f = lowerVal.floatValue(); - if (!includeUpper && f > 0.0f && Float.isInfinite(f)) - return null; - int i = NumericUtils.floatToSortableInt(f); - inclusiveLowerPoint = NumericUtils.sortableIntToFloat( includeLower ? i : (i + 1) ); - } else { - inclusiveLowerPoint = Float.NEGATIVE_INFINITY; - } - if (upperVal != null) { - float f = upperVal.floatValue(); - if (!includeUpper && f < 0.0f && Float.isInfinite(f)) - return null; - int i = NumericUtils.floatToSortableInt(f); - inclusiveUpperPoint = NumericUtils.sortableIntToFloat( includeUpper ? i : (i - 1) ); - } else { - inclusiveUpperPoint = Float.POSITIVE_INFINITY; - } - - if (inclusiveLowerPoint > inclusiveUpperPoint) - return null; - - final NumericDocValues values = DocValues.getNumeric(context.reader(), field); - return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) { - @Override - protected boolean matchDoc(int doc) { - final float value = Float.intBitsToFloat((int)values.get(doc)); - return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint; - } - }; - } - }; - } - - /** - * Creates a numeric range filter using {@link org.apache.lucene.index.LeafReader#getNumericDocValues(String)}. This works with all - * double fields containing exactly one numeric term in the field. The range can be half-open by setting one - * of the values to null. - */ - public static DocValuesRangeFilter newDoubleRange(String field, Double lowerVal, Double upperVal, boolean includeLower, boolean includeUpper) { - return new DocValuesRangeFilter(field, lowerVal, upperVal, includeLower, includeUpper) { - @Override - public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException { - // we transform the floating point numbers to sortable integers - // using NumericUtils to easier find the next bigger/lower value - final double inclusiveLowerPoint, inclusiveUpperPoint; - if (lowerVal != null) { - double f = lowerVal.doubleValue(); - if (!includeUpper && f > 0.0 && Double.isInfinite(f)) - return null; - long i = NumericUtils.doubleToSortableLong(f); - inclusiveLowerPoint = NumericUtils.sortableLongToDouble( includeLower ? i : (i + 1L) ); - } else { - inclusiveLowerPoint = Double.NEGATIVE_INFINITY; - } - if (upperVal != null) { - double f = upperVal.doubleValue(); - if (!includeUpper && f < 0.0 && Double.isInfinite(f)) - return null; - long i = NumericUtils.doubleToSortableLong(f); - inclusiveUpperPoint = NumericUtils.sortableLongToDouble( includeUpper ? i : (i - 1L) ); - } else { - inclusiveUpperPoint = Double.POSITIVE_INFINITY; - } - - if (inclusiveLowerPoint > inclusiveUpperPoint) - return null; - - final NumericDocValues values = DocValues.getNumeric(context.reader(), field); - // ignore deleted docs if range doesn't contain 0 - return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) { - @Override - protected boolean matchDoc(int doc) { - final double value = Double.longBitsToDouble(values.get(doc)); - return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint; - } - }; - } - }; - } - - @Override - public final String toString(String defaultField) { - final StringBuilder sb = new StringBuilder(field).append(":"); - return sb.append(includeLower ? '[' : '{') - .append((lowerVal == null) ? "*" : lowerVal.toString()) - .append(" TO ") - .append((upperVal == null) ? "*" : upperVal.toString()) - .append(includeUpper ? ']' : '}') - .toString(); - } - - @Override - @SuppressWarnings({"rawtypes"}) - public final boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof DocValuesRangeFilter)) return false; - DocValuesRangeFilter other = (DocValuesRangeFilter) o; - - if (!this.field.equals(other.field) - || this.includeLower != other.includeLower - || this.includeUpper != other.includeUpper - ) { return false; } - if (this.lowerVal != null ? !this.lowerVal.equals(other.lowerVal) : other.lowerVal != null) return false; - if (this.upperVal != null ? !this.upperVal.equals(other.upperVal) : other.upperVal != null) return false; - return true; - } - - @Override - public final int hashCode() { - int h = field.hashCode(); - h ^= (lowerVal != null) ? lowerVal.hashCode() : 550356204; - h = (h << 1) | (h >>> 31); // rotate to distinguish lower from upper - h ^= (upperVal != null) ? upperVal.hashCode() : -1674416163; - h ^= (includeLower ? 1549299360 : -365038026) ^ (includeUpper ? 1721088258 : 1948649653); - return h; - } - - /** Returns the field name for this filter */ - public String getField() { return field; } - - /** Returns true if the lower endpoint is inclusive */ - public boolean includesLower() { return includeLower; } - - /** Returns true if the upper endpoint is inclusive */ - public boolean includesUpper() { return includeUpper; } - - /** Returns the lower value of this range filter */ - public T getLowerVal() { return lowerVal; } - - /** Returns the upper value of this range filter */ - public T getUpperVal() { return upperVal; } -} diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeQuery.java new file mode 100644 index 00000000000..c72b56061b3 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeQuery.java @@ -0,0 +1,393 @@ +package org.apache.lucene.search; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Objects; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.Bits.MatchNoBits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.ToStringUtils; + +/** + * A range query that works on top of the doc values APIs. Such queries are + * usually slow since they do not use an inverted index. However, in the + * dense case where most documents match this query, it might be as + * fast or faster than a regular {@link NumericRangeQuery}. + * @lucene.experimental + */ +public final class DocValuesRangeQuery extends Query { + + /** Create a new numeric range query on a numeric doc-values field. The field + * must has been indexed with either {@link DocValuesType#NUMERIC} or + * {@link DocValuesType#SORTED_NUMERIC} doc values. */ + public static Query newLongRange(String field, Long lowerVal, Long upperVal, boolean includeLower, boolean includeUpper) { + return new DocValuesRangeQuery(field, lowerVal, upperVal, includeLower, includeUpper); + } + + /** Create a new numeric range query on a numeric doc-values field. The field + * must has been indexed with {@link DocValuesType#SORTED} or + * {@link DocValuesType#SORTED_SET} doc values. */ + public static Query newBytesRefRange(String field, BytesRef lowerVal, BytesRef upperVal, boolean includeLower, boolean includeUpper) { + return new DocValuesRangeQuery(field, deepCopyOf(lowerVal), deepCopyOf(upperVal), includeLower, includeUpper); + } + + private static BytesRef deepCopyOf(BytesRef b) { + if (b == null) { + return null; + } else { + return BytesRef.deepCopyOf(b); + } + } + + private final String field; + private final Object lowerVal, upperVal; + private final boolean includeLower, includeUpper; + + private DocValuesRangeQuery(String field, Object lowerVal, Object upperVal, boolean includeLower, boolean includeUpper) { + this.field = Objects.requireNonNull(field); + this.lowerVal = lowerVal; + this.upperVal = upperVal; + this.includeLower = includeLower; + this.includeUpper = includeUpper; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof DocValuesRangeQuery == false) { + return false; + } + final DocValuesRangeQuery that = (DocValuesRangeQuery) obj; + return field.equals(that.field) + && Objects.equals(lowerVal, that.lowerVal) + && Objects.equals(upperVal, that.upperVal) + && includeLower == that.includeLower + && includeUpper == that.includeUpper + && getBoost() == that.getBoost(); + } + + @Override + public int hashCode() { + return Objects.hash(field, lowerVal, upperVal, includeLower, includeUpper, getBoost()); + } + + @Override + public String toString(String field) { + StringBuilder sb = new StringBuilder(); + if (this.field.equals(field) == false) { + sb.append(this.field).append(':'); + } + sb.append(includeLower ? '[' : '{'); + sb.append(lowerVal == null ? "*" : lowerVal.toString()); + sb.append(" TO "); + sb.append(upperVal == null ? "*" : upperVal.toString()); + sb.append(includeUpper ? ']' : '}'); + sb.append(ToStringUtils.boost(getBoost())); + return sb.toString(); + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + if (lowerVal == null && upperVal == null) { + final FieldValueQuery rewritten = new FieldValueQuery(field); + rewritten.setBoost(getBoost()); + return rewritten; + } + return this; + } + + @Override + public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + if (lowerVal == null && upperVal == null) { + throw new IllegalStateException("Both min and max values cannot be null, call rewrite first"); + } + return new Weight(DocValuesRangeQuery.this) { + + private float queryNorm; + private float queryWeight; + + @Override + public float getValueForNormalization() throws IOException { + queryWeight = getBoost(); + return queryWeight * queryWeight; + } + + @Override + public void normalize(float norm, float topLevelBoost) { + queryNorm = norm * topLevelBoost; + queryWeight *= queryNorm; + } + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + final Scorer s = scorer(context, context.reader().getLiveDocs()); + final boolean exists = (s != null && s.advance(doc) == doc); + + final ComplexExplanation result = new ComplexExplanation(); + if (exists) { + result.setDescription(DocValuesRangeQuery.this.toString() + ", product of:"); + result.setValue(queryWeight); + result.setMatch(Boolean.TRUE); + result.addDetail(new Explanation(getBoost(), "boost")); + result.addDetail(new Explanation(queryNorm, "queryNorm")); + } else { + result.setDescription(DocValuesRangeQuery.this.toString() + " doesn't match id " + doc); + result.setValue(0); + result.setMatch(Boolean.FALSE); + } + return result; + } + + @Override + public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + + final Bits docsWithField = context.reader().getDocsWithField(field); + if (docsWithField == null || docsWithField instanceof MatchNoBits) { + return null; + } + + final DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); + final TwoPhaseDocIdSetIterator twoPhaseRange; + if (lowerVal instanceof Long || upperVal instanceof Long) { + + final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field); + + final long min; + if (lowerVal == null) { + min = Long.MIN_VALUE; + } else if (includeLower) { + min = (long) lowerVal; + } else { + min = 1 + (long) lowerVal; + } + + final long max; + if (upperVal == null) { + max = Long.MAX_VALUE; + } else if (includeUpper) { + max = (long) upperVal; + } else { + max = -1 + (long) upperVal; + } + + if (min > max) { + return null; + } + + twoPhaseRange = new TwoPhaseNumericRange(values, min, max, approximation, acceptDocs); + + } else if (lowerVal instanceof BytesRef || upperVal instanceof BytesRef) { + + final SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); + + final long minOrd; + if (lowerVal == null) { + minOrd = 0; + } else { + final long ord = values.lookupTerm((BytesRef) lowerVal); + if (ord < 0) { + minOrd = -1 - ord; + } else if (includeLower) { + minOrd = ord; + } else { + minOrd = ord + 1; + } + } + + final long maxOrd; + if (upperVal == null) { + maxOrd = values.getValueCount() - 1; + } else { + final long ord = values.lookupTerm((BytesRef) upperVal); + if (ord < 0) { + maxOrd = -2 - ord; + } else if (includeUpper) { + maxOrd = ord; + } else { + maxOrd = ord - 1; + } + } + + if (minOrd > maxOrd) { + return null; + } + + twoPhaseRange = new TwoPhaseOrdRange(values, minOrd, maxOrd, approximation, acceptDocs); + + } else { + throw new AssertionError(); + } + + return new RangeScorer(this, twoPhaseRange, queryWeight); + } + + }; + } + + private static class TwoPhaseNumericRange extends TwoPhaseDocIdSetIterator { + + private final DocIdSetIterator approximation; + private final SortedNumericDocValues values; + private final long min, max; + private final Bits acceptDocs; + + TwoPhaseNumericRange(SortedNumericDocValues values, long min, long max, DocIdSetIterator approximation, Bits acceptDocs) { + this.values = values; + this.min = min; + this.max = max; + this.approximation = approximation; + this.acceptDocs = acceptDocs; + } + + @Override + public DocIdSetIterator approximation() { + return approximation; + } + + @Override + public boolean matches() throws IOException { + final int doc = approximation.docID(); + if (acceptDocs == null || acceptDocs.get(doc)) { + values.setDocument(doc); + final int count = values.count(); + for (int i = 0; i < count; ++i) { + final long value = values.valueAt(i); + if (value >= min && value <= max) { + return true; + } + } + } + return false; + } + + } + + private static class TwoPhaseOrdRange extends TwoPhaseDocIdSetIterator { + + private final DocIdSetIterator approximation; + private final SortedSetDocValues values; + private final long minOrd, maxOrd; + private final Bits acceptDocs; + + TwoPhaseOrdRange(SortedSetDocValues values, long minOrd, long maxOrd, DocIdSetIterator approximation, Bits acceptDocs) { + this.values = values; + this.minOrd = minOrd; + this.maxOrd = maxOrd; + this.approximation = approximation; + this.acceptDocs = acceptDocs; + } + + @Override + public DocIdSetIterator approximation() { + return approximation; + } + + @Override + public boolean matches() throws IOException { + final int doc = approximation.docID(); + if (acceptDocs == null || acceptDocs.get(doc)) { + values.setDocument(doc); + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + if (ord >= minOrd && ord <= maxOrd) { + return true; + } + } + } + return false; + } + + } + + private static class RangeScorer extends Scorer { + + private final TwoPhaseDocIdSetIterator twoPhaseRange; + private final DocIdSetIterator disi; + private final float score; + + RangeScorer(Weight weight, TwoPhaseDocIdSetIterator twoPhaseRange, float score) { + super(weight); + this.twoPhaseRange = twoPhaseRange; + this.disi = TwoPhaseDocIdSetIterator.asDocIdSetIterator(twoPhaseRange); + this.score = score; + } + + @Override + public TwoPhaseDocIdSetIterator asTwoPhaseIterator() { + return twoPhaseRange; + } + + @Override + public float score() throws IOException { + return score; + } + + @Override + public int freq() throws IOException { + return 1; + } + + @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public int startOffset() throws IOException { + return -1; + } + + @Override + public int endOffset() throws IOException { + return -1; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; + } + + @Override + public int docID() { + return disi.docID(); + } + + @Override + public int nextDoc() throws IOException { + return disi.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + return disi.advance(target); + } + + @Override + public long cost() { + return disi.cost(); + } + + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java b/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java deleted file mode 100644 index f52008e513f..00000000000 --- a/lucene/core/src/java/org/apache/lucene/search/FieldValueFilter.java +++ /dev/null @@ -1,145 +0,0 @@ -package org.apache.lucene.search; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.util.BitDocIdSet; -import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.Bits.MatchAllBits; -import org.apache.lucene.util.Bits.MatchNoBits; - -/** - * A {@link Filter} that accepts all documents that have one or more values in a - * given field. This {@link Filter} request {@link Bits} from - * {@link org.apache.lucene.index.LeafReader#getDocsWithField} - */ -public class FieldValueFilter extends Filter { - private final String field; - private final boolean negate; - - /** - * Creates a new {@link FieldValueFilter} - * - * @param field - * the field to filter - */ - public FieldValueFilter(String field) { - this(field, false); - } - - /** - * Creates a new {@link FieldValueFilter} - * - * @param field - * the field to filter - * @param negate - * iff true all documents with no value in the given - * field are accepted. - * - */ - public FieldValueFilter(String field, boolean negate) { - this.field = field; - this.negate = negate; - } - - /** - * Returns the field this filter is applied on. - * @return the field this filter is applied on. - */ - public String field() { - return field; - } - - /** - * Returns true iff this filter is negated, otherwise false - * @return true iff this filter is negated, otherwise false - */ - public boolean negate() { - return negate; - } - - @Override - public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) - throws IOException { - final Bits docsWithField = DocValues.getDocsWithField( - context.reader(), field); - if (negate) { - if (docsWithField instanceof MatchAllBits) { - return null; - } - return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) { - @Override - protected final boolean matchDoc(int doc) { - return !docsWithField.get(doc); - } - }; - } else { - if (docsWithField instanceof MatchNoBits) { - return null; - } - if (docsWithField instanceof BitSet) { - // UweSays: this is always the case for our current impl - but who knows - // :-) - return BitsFilteredDocIdSet.wrap(new BitDocIdSet((BitSet) docsWithField), acceptDocs); - } - return new DocValuesDocIdSet(context.reader().maxDoc(), acceptDocs) { - @Override - protected final boolean matchDoc(int doc) { - return docsWithField.get(doc); - } - }; - } - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((field == null) ? 0 : field.hashCode()); - result = prime * result + (negate ? 1231 : 1237); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - FieldValueFilter other = (FieldValueFilter) obj; - if (field == null) { - if (other.field != null) - return false; - } else if (!field.equals(other.field)) - return false; - if (negate != other.negate) - return false; - return true; - } - - @Override - public String toString(String defaultField) { - return "FieldValueFilter [field=" + field + ", negate=" + negate + "]"; - } - -} diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldValueQuery.java b/lucene/core/src/java/org/apache/lucene/search/FieldValueQuery.java new file mode 100644 index 00000000000..7cd679f00ac --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/FieldValueQuery.java @@ -0,0 +1,192 @@ +package org.apache.lucene.search; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Objects; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.Bits.MatchNoBits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.ToStringUtils; + +/** + * A {@link Query} that matches documents that have a value for a given field + * as reported by {@link LeafReader#getDocsWithField(String)}. + */ +public final class FieldValueQuery extends Query { + + private final String field; + + /** Create a query that will match that have a value for the given + * {@code field}. */ + public FieldValueQuery(String field) { + this.field = Objects.requireNonNull(field); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof FieldValueQuery == false) { + return false; + } + final FieldValueQuery that = (FieldValueQuery) obj; + return field.equals(that.field) && getBoost() == that.getBoost(); + } + + @Override + public int hashCode() { + return Objects.hash(getClass(), field, getBoost()); + } + + @Override + public String toString(String field) { + return "FieldValueQuery [field=" + this.field + "]" + ToStringUtils.boost(getBoost()); + } + + @Override + public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + return new Weight(this) { + + private float queryNorm; + private float queryWeight; + + @Override + public float getValueForNormalization() throws IOException { + queryWeight = getBoost(); + return queryWeight * queryWeight; + } + + @Override + public void normalize(float norm, float topLevelBoost) { + queryNorm = norm * topLevelBoost; + queryWeight *= queryNorm; + } + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + final Scorer s = scorer(context, context.reader().getLiveDocs()); + final boolean exists = (s != null && s.advance(doc) == doc); + + final ComplexExplanation result = new ComplexExplanation(); + if (exists) { + result.setDescription(FieldValueQuery.this.toString() + ", product of:"); + result.setValue(queryWeight); + result.setMatch(Boolean.TRUE); + result.addDetail(new Explanation(getBoost(), "boost")); + result.addDetail(new Explanation(queryNorm, "queryNorm")); + } else { + result.setDescription(FieldValueQuery.this.toString() + " doesn't match id " + doc); + result.setValue(0); + result.setMatch(Boolean.FALSE); + } + return result; + } + + @Override + public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + final Bits docsWithField = context.reader().getDocsWithField(field); + if (docsWithField == null || docsWithField instanceof MatchNoBits) { + return null; + } + + final DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); + final TwoPhaseDocIdSetIterator twoPhaseIterator = new TwoPhaseDocIdSetIterator() { + + @Override + public boolean matches() throws IOException { + final int doc = approximation.docID(); + if (acceptDocs != null && acceptDocs.get(doc) == false) { + return false; + } + if (docsWithField.get(doc) == false) { + return false; + } + return true; + } + + @Override + public DocIdSetIterator approximation() { + return approximation; + } + }; + final DocIdSetIterator disi = TwoPhaseDocIdSetIterator.asDocIdSetIterator(twoPhaseIterator); + + return new Scorer(this) { + + @Override + public TwoPhaseDocIdSetIterator asTwoPhaseIterator() { + return twoPhaseIterator; + } + + @Override + public int nextDoc() throws IOException { + return disi.nextDoc(); + } + + @Override + public int docID() { + return disi.docID(); + } + + @Override + public long cost() { + return disi.cost(); + } + + @Override + public int advance(int target) throws IOException { + return disi.advance(target); + } + + @Override + public int startOffset() throws IOException { + return -1; + } + + @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; + } + + @Override + public int freq() throws IOException { + return 1; + } + + @Override + public int endOffset() throws IOException { + return -1; + } + + @Override + public float score() throws IOException { + return queryWeight; + } + }; + } + }; + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/search/TermRangeFilter.java b/lucene/core/src/java/org/apache/lucene/search/TermRangeFilter.java index 9789a328ad3..2001d5a33e3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermRangeFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermRangeFilter.java @@ -27,9 +27,6 @@ import org.apache.lucene.util.BytesRef; * supplied range according to {@link * Byte#compareTo(Byte)}, It is not intended * for numerical ranges; use {@link NumericRangeFilter} instead. - * - *

If you construct a large number of range filters with different ranges but on the - * same field, {@link DocValuesRangeFilter} may have significantly better performance. * @since 2.9 */ public class TermRangeFilter extends MultiTermQueryWrapperFilter { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java b/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java index 2238fc2cac2..c1ed29f6757 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java @@ -268,7 +268,7 @@ public class TestCachingWrapperFilter extends LuceneTestCase { // returns default empty docidset, always cacheable: assertDocIdSetCacheable(reader, NumericRangeFilter.newIntRange("test", Integer.valueOf(10000), Integer.valueOf(-10000), true, true), true); // is cacheable: - assertDocIdSetCacheable(reader, DocValuesRangeFilter.newIntRange("test", Integer.valueOf(10), Integer.valueOf(20), true, true), false); + assertDocIdSetCacheable(reader, NumericRangeFilter.newIntRange("test", 10, 20, true, true), false); // a fixedbitset filter is always cacheable assertDocIdSetCacheable(reader, new Filter() { @Override diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocTermOrdsRangeFilter.java b/lucene/core/src/test/org/apache/lucene/search/TestDocTermOrdsRangeFilter.java deleted file mode 100644 index 7f393faf476..00000000000 --- a/lucene/core/src/test/org/apache/lucene/search/TestDocTermOrdsRangeFilter.java +++ /dev/null @@ -1,126 +0,0 @@ -package org.apache.lucene.search; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.SortedSetDocValuesField; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; -import org.apache.lucene.util.UnicodeUtil; - -/** - * Tests the DocTermOrdsRangeFilter - */ -public class TestDocTermOrdsRangeFilter extends LuceneTestCase { - protected IndexSearcher searcher1; - protected IndexSearcher searcher2; - private IndexReader reader; - private Directory dir; - protected String fieldName; - - @Override - public void setUp() throws Exception { - super.setUp(); - dir = newDirectory(); - fieldName = random().nextBoolean() ? "field" : ""; // sometimes use an empty string as field name - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, - newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)) - .setMaxBufferedDocs(TestUtil.nextInt(random(), 50, 1000))); - List terms = new ArrayList<>(); - int num = atLeast(200); - for (int i = 0; i < num; i++) { - Document doc = new Document(); - doc.add(newStringField("id", Integer.toString(i), Field.Store.NO)); - int numTerms = random().nextInt(4); - for (int j = 0; j < numTerms; j++) { - String s = TestUtil.randomUnicodeString(random()); - doc.add(newStringField(fieldName, s, Field.Store.NO)); - doc.add(new SortedSetDocValuesField(fieldName, new BytesRef(s))); - terms.add(s); - } - writer.addDocument(doc); - } - - if (VERBOSE) { - // utf16 order - Collections.sort(terms); - System.out.println("UTF16 order:"); - for(String s : terms) { - System.out.println(" " + UnicodeUtil.toHexString(s)); - } - } - - int numDeletions = random().nextInt(num/10); - for (int i = 0; i < numDeletions; i++) { - writer.deleteDocuments(new Term("id", Integer.toString(random().nextInt(num)))); - } - - reader = writer.getReader(); - searcher1 = newSearcher(reader); - searcher2 = newSearcher(reader); - writer.close(); - } - - @Override - public void tearDown() throws Exception { - reader.close(); - dir.close(); - super.tearDown(); - } - - /** test a bunch of random ranges */ - public void testRanges() throws Exception { - int num = atLeast(1000); - for (int i = 0; i < num; i++) { - BytesRef lowerVal = new BytesRef(TestUtil.randomUnicodeString(random())); - BytesRef upperVal = new BytesRef(TestUtil.randomUnicodeString(random())); - if (upperVal.compareTo(lowerVal) < 0) { - assertSame(upperVal, lowerVal, random().nextBoolean(), random().nextBoolean()); - } else { - assertSame(lowerVal, upperVal, random().nextBoolean(), random().nextBoolean()); - } - } - } - - /** check that the # of hits is the same as if the query - * is run against the inverted index - */ - protected void assertSame(BytesRef lowerVal, BytesRef upperVal, boolean includeLower, boolean includeUpper) throws IOException { - Query docValues = new ConstantScoreQuery(DocTermOrdsRangeFilter.newBytesRefRange(fieldName, lowerVal, upperVal, includeLower, includeUpper)); - MultiTermQuery inverted = new TermRangeQuery(fieldName, lowerVal, upperVal, includeLower, includeUpper); - inverted.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE); - - TopDocs invertedDocs = searcher1.search(inverted, 25); - TopDocs docValuesDocs = searcher2.search(docValues, 25); - - CheckHits.checkEqual(inverted, invertedDocs.scoreDocs, docValuesDocs.scoreDocs); - } -} diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java new file mode 100644 index 00000000000..b3ddb130da2 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java @@ -0,0 +1,283 @@ +package org.apache.lucene.search; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.TestUtil; + +public class TestDocValuesRangeQuery extends LuceneTestCase { + + public void testDuelNumericRangeQuery() throws IOException { + final int iters = atLeast(10); + for (int iter = 0; iter < iters; ++iter) { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + final int numDocs = atLeast(100); + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + final int numValues = random().nextInt(2); + for (int j = 0; j < numValues; ++j) { + final long value = TestUtil.nextLong(random(), -100, 10000); + doc.add(new SortedNumericDocValuesField("dv", value)); + doc.add(new LongField("idx", value, Store.NO)); + } + iw.addDocument(doc); + } + if (random().nextBoolean()) { + iw.deleteDocuments(NumericRangeQuery.newLongRange("idx", 0L, 10L, true, true)); + } + iw.commit(); + final IndexReader reader = iw.getReader(); + final IndexSearcher searcher = newSearcher(reader); + iw.close(); + + for (int i = 0; i < 100; ++i) { + final Long min = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); + final Long max = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); + final boolean minInclusive = random().nextBoolean(); + final boolean maxInclusive = random().nextBoolean(); + final Query q1 = NumericRangeQuery.newLongRange("idx", min, max, minInclusive, maxInclusive); + final Query q2 = DocValuesRangeQuery.newLongRange("dv", min, max, minInclusive, maxInclusive); + assertSameMatches(searcher, q1, q2, false); + } + + reader.close(); + dir.close(); + } + } + + private static BytesRef toSortableBytes(Long l) { + if (l == null) { + return null; + } else { + final BytesRefBuilder bytes = new BytesRefBuilder(); + NumericUtils.longToPrefixCoded(l, 0, bytes); + return bytes.get(); + } + } + + public void testDuelNumericSorted() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + final int numDocs = atLeast(100); + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + final int numValues = random().nextInt(3); + for (int j = 0; j < numValues; ++j) { + final long value = TestUtil.nextLong(random(), -100, 10000); + doc.add(new SortedNumericDocValuesField("dv1", value)); + doc.add(new SortedSetDocValuesField("dv2", toSortableBytes(value))); + } + iw.addDocument(doc); + } + if (random().nextBoolean()) { + iw.deleteDocuments(DocValuesRangeQuery.newLongRange("dv1", 0L, 10L, true, true)); + } + iw.commit(); + final IndexReader reader = iw.getReader(); + final IndexSearcher searcher = newSearcher(reader); + iw.close(); + + for (int i = 0; i < 100; ++i) { + final Long min = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); + final Long max = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); + final boolean minInclusive = random().nextBoolean(); + final boolean maxInclusive = random().nextBoolean(); + final Query q1 = DocValuesRangeQuery.newLongRange("dv1", min, max, minInclusive, maxInclusive); + final Query q2 = DocValuesRangeQuery.newBytesRefRange("dv2", toSortableBytes(min), toSortableBytes(max), minInclusive, maxInclusive); + assertSameMatches(searcher, q1, q2, true); + } + + reader.close(); + dir.close(); + } + + public void testScore() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + final int numDocs = atLeast(100); + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + final int numValues = random().nextInt(3); + for (int j = 0; j < numValues; ++j) { + final long value = TestUtil.nextLong(random(), -100, 10000); + doc.add(new SortedNumericDocValuesField("dv1", value)); + doc.add(new SortedSetDocValuesField("dv2", toSortableBytes(value))); + } + iw.addDocument(doc); + } + if (random().nextBoolean()) { + iw.deleteDocuments(DocValuesRangeQuery.newLongRange("dv1", 0L, 10L, true, true)); + } + iw.commit(); + final IndexReader reader = iw.getReader(); + final IndexSearcher searcher = newSearcher(reader); + iw.close(); + + for (int i = 0; i < 100; ++i) { + final Long min = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); + final Long max = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); + final boolean minInclusive = random().nextBoolean(); + final boolean maxInclusive = random().nextBoolean(); + + final float boost = random().nextFloat() * 10; + + final Query q1 = DocValuesRangeQuery.newLongRange("dv1", min, max, minInclusive, maxInclusive); + q1.setBoost(boost); + final ConstantScoreQuery csq1 = new ConstantScoreQuery(DocValuesRangeQuery.newLongRange("dv1", min, max, minInclusive, maxInclusive)); + csq1.setBoost(boost); + assertSameMatches(searcher, q1, csq1, true); + + final Query q2 = DocValuesRangeQuery.newBytesRefRange("dv2", toSortableBytes(min), toSortableBytes(max), minInclusive, maxInclusive); + q2.setBoost(boost); + final ConstantScoreQuery csq2 = new ConstantScoreQuery(DocValuesRangeQuery.newBytesRefRange("dv2", toSortableBytes(min), toSortableBytes(max), minInclusive, maxInclusive)); + csq2.setBoost(boost); + assertSameMatches(searcher, q2, csq2, true); + } + + reader.close(); + dir.close(); + } + + public void testApproximation() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + final int numDocs = atLeast(100); + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + final int numValues = random().nextInt(3); + for (int j = 0; j < numValues; ++j) { + final long value = TestUtil.nextLong(random(), -100, 10000); + doc.add(new SortedNumericDocValuesField("dv1", value)); + doc.add(new SortedSetDocValuesField("dv2", toSortableBytes(value))); + doc.add(new LongField("idx", value, Store.NO)); + doc.add(new StringField("f", random().nextBoolean() ? "a" : "b", Store.NO)); + } + iw.addDocument(doc); + } + if (random().nextBoolean()) { + iw.deleteDocuments(NumericRangeQuery.newLongRange("idx", 0L, 10L, true, true)); + } + iw.commit(); + final IndexReader reader = iw.getReader(); + final IndexSearcher searcher = newSearcher(reader); + iw.close(); + + for (int i = 0; i < 100; ++i) { + final Long min = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); + final Long max = random().nextBoolean() ? null : TestUtil.nextLong(random(), -100, 1000); + final boolean minInclusive = random().nextBoolean(); + final boolean maxInclusive = random().nextBoolean(); + + BooleanQuery ref = new BooleanQuery(); + ref.add(NumericRangeQuery.newLongRange("idx", min, max, minInclusive, maxInclusive), Occur.FILTER); + ref.add(new TermQuery(new Term("f", "a")), Occur.MUST); + + BooleanQuery bq1 = new BooleanQuery(); + bq1.add(DocValuesRangeQuery.newLongRange("dv1", min, max, minInclusive, maxInclusive), Occur.FILTER); + bq1.add(new TermQuery(new Term("f", "a")), Occur.MUST); + + assertSameMatches(searcher, ref, bq1, true); + + BooleanQuery bq2 = new BooleanQuery(); + bq2.add(DocValuesRangeQuery.newBytesRefRange("dv2", toSortableBytes(min), toSortableBytes(max), minInclusive, maxInclusive), Occur.FILTER); + bq2.add(new TermQuery(new Term("f", "a")), Occur.MUST); + + assertSameMatches(searcher, ref, bq2, true); + } + + reader.close(); + dir.close(); + } + + private void assertSameMatches(IndexSearcher searcher, Query q1, Query q2, boolean scores) throws IOException { + final int maxDoc = searcher.getIndexReader().maxDoc(); + final TopDocs td1 = searcher.search(q1, maxDoc, scores ? Sort.RELEVANCE : Sort.INDEXORDER); + final TopDocs td2 = searcher.search(q2, maxDoc, scores ? Sort.RELEVANCE : Sort.INDEXORDER); + assertEquals(td1.totalHits, td2.totalHits); + for (int i = 0; i < td1.scoreDocs.length; ++i) { + assertEquals(td1.scoreDocs[i].doc, td2.scoreDocs[i].doc); + if (scores) { + assertEquals(td1.scoreDocs[i].score, td2.scoreDocs[i].score, 10e-7); + } + } + } + + public void testToString() { + assertEquals("f:[2 TO 5]", DocValuesRangeQuery.newLongRange("f", 2L, 5L, true, true).toString()); + assertEquals("f:{2 TO 5]", DocValuesRangeQuery.newLongRange("f", 2L, 5L, false, true).toString()); + assertEquals("f:{2 TO 5}", DocValuesRangeQuery.newLongRange("f", 2L, 5L, false, false).toString()); + assertEquals("f:{* TO 5}", DocValuesRangeQuery.newLongRange("f", null, 5L, false, false).toString()); + assertEquals("f:[2 TO *}", DocValuesRangeQuery.newLongRange("f", 2L, null, true, false).toString()); + + BytesRef min = new BytesRef("a"); + BytesRef max = new BytesRef("b"); + assertEquals("f:[[61] TO [62]]", DocValuesRangeQuery.newBytesRefRange("f", min, max, true, true).toString()); + assertEquals("f:{[61] TO [62]]", DocValuesRangeQuery.newBytesRefRange("f", min, max, false, true).toString()); + assertEquals("f:{[61] TO [62]}", DocValuesRangeQuery.newBytesRefRange("f", min, max, false, false).toString()); + assertEquals("f:{* TO [62]}", DocValuesRangeQuery.newBytesRefRange("f", null, max, false, false).toString()); + assertEquals("f:[[61] TO *}", DocValuesRangeQuery.newBytesRefRange("f", min, null, true, false).toString()); + } + + public void testDocValuesRangeSupportsApproximation() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new NumericDocValuesField("dv1", 5L)); + doc.add(new SortedDocValuesField("dv2", toSortableBytes(42L))); + iw.addDocument(doc); + iw.commit(); + final IndexReader reader = iw.getReader(); + final LeafReaderContext ctx = reader.leaves().get(0); + final IndexSearcher searcher = newSearcher(reader); + iw.close(); + + Query q1 = DocValuesRangeQuery.newLongRange("dv1", 0L, 100L, random().nextBoolean(), random().nextBoolean()); + Weight w = searcher.createNormalizedWeight(q1, random().nextBoolean()); + Scorer s = w.scorer(ctx, null); + assertNotNull(s.asTwoPhaseIterator()); + + Query q2 = DocValuesRangeQuery.newBytesRefRange("dv2", toSortableBytes(0L), toSortableBytes(100L), random().nextBoolean(), random().nextBoolean()); + w = searcher.createNormalizedWeight(q2, random().nextBoolean()); + s = w.scorer(ctx, null); + assertNotNull(s.asTwoPhaseIterator()); + + reader.close(); + dir.close(); + } + +} diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFieldCacheRangeFilter.java b/lucene/core/src/test/org/apache/lucene/search/TestFieldCacheRangeFilter.java deleted file mode 100644 index 098f599d6d1..00000000000 --- a/lucene/core/src/test/org/apache/lucene/search/TestFieldCacheRangeFilter.java +++ /dev/null @@ -1,480 +0,0 @@ -package org.apache.lucene.search; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.IntField; -import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.NumericUtils; -import org.junit.Test; - -/** - * A basic 'positive' Unit test class for the FieldCacheRangeFilter class. - * - *

- * NOTE: at the moment, this class only tests for 'positive' results, - * it does not verify the results to ensure there are no 'false positives', - * nor does it adequately test 'negative' results. It also does not test - * that garbage in results in an Exception. - */ -public class TestFieldCacheRangeFilter extends BaseTestRangeFilter { - - @Test - public void testRangeFilterId() throws IOException { - - IndexReader reader = signedIndexReader; - IndexSearcher search = newSearcher(reader); - - int medId = ((maxId - minId) / 2); - - String minIP = pad(minId); - String maxIP = pad(maxId); - String medIP = pad(medId); - - int numDocs = reader.numDocs(); - - assertEquals("num of docs", numDocs, 1+ maxId - minId); - - ScoreDoc[] result; - Query q = new TermQuery(new Term("body","body")); - - // test id, bounded on both ends - result = search.search(q, DocValuesRangeFilter.newStringRange("id",minIP,maxIP,T,T), numDocs).scoreDocs; - assertEquals("find all", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,maxIP,T,F), numDocs).scoreDocs; - assertEquals("all but last", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,maxIP,F,T), numDocs).scoreDocs; - assertEquals("all but first", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,maxIP,F,F), numDocs).scoreDocs; - assertEquals("all but ends", numDocs-2, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",medIP,maxIP,T,T), numDocs).scoreDocs; - assertEquals("med and up", 1+ maxId-medId, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,medIP,T,T), numDocs).scoreDocs; - assertEquals("up to med", 1+ medId-minId, result.length); - - // unbounded id - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",null,null,T,T), numDocs).scoreDocs; - assertEquals("find all", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,null,T,F), numDocs).scoreDocs; - assertEquals("min and up", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",null,maxIP,F,T), numDocs).scoreDocs; - assertEquals("max and down", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,null,F,F), numDocs).scoreDocs; - assertEquals("not min, but up", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",null,maxIP,F,F), numDocs).scoreDocs; - assertEquals("not max, but down", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",medIP,maxIP,T,F), numDocs).scoreDocs; - assertEquals("med and up, not max", maxId-medId, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,medIP,F,T), numDocs).scoreDocs; - assertEquals("not min, up to med", medId-minId, result.length); - - // very small sets - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,minIP,F,F), numDocs).scoreDocs; - assertEquals("min,min,F,F", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newStringRange("id",medIP,medIP,F,F), numDocs).scoreDocs; - assertEquals("med,med,F,F", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newStringRange("id",maxIP,maxIP,F,F), numDocs).scoreDocs; - assertEquals("max,max,F,F", 0, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",minIP,minIP,T,T), numDocs).scoreDocs; - assertEquals("min,min,T,T", 1, result.length); - result = search.search(q,DocValuesRangeFilter.newStringRange("id",null,minIP,F,T), numDocs).scoreDocs; - assertEquals("nul,min,F,T", 1, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",maxIP,maxIP,T,T), numDocs).scoreDocs; - assertEquals("max,max,T,T", 1, result.length); - result = search.search(q,DocValuesRangeFilter.newStringRange("id",maxIP,null,T,F), numDocs).scoreDocs; - assertEquals("max,nul,T,T", 1, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("id",medIP,medIP,T,T), numDocs).scoreDocs; - assertEquals("med,med,T,T", 1, result.length); - } - - @Test - public void testFieldCacheRangeFilterRand() throws IOException { - - IndexReader reader = signedIndexReader; - IndexSearcher search = newSearcher(reader); - - String minRP = pad(signedIndexDir.minR); - String maxRP = pad(signedIndexDir.maxR); - - int numDocs = reader.numDocs(); - - assertEquals("num of docs", numDocs, 1+ maxId - minId); - - ScoreDoc[] result; - Query q = new TermQuery(new Term("body","body")); - - // test extremes, bounded on both ends - - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,maxRP,T,T), numDocs).scoreDocs; - assertEquals("find all", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,maxRP,T,F), numDocs).scoreDocs; - assertEquals("all but biggest", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,maxRP,F,T), numDocs).scoreDocs; - assertEquals("all but smallest", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,maxRP,F,F), numDocs).scoreDocs; - assertEquals("all but extremes", numDocs-2, result.length); - - // unbounded - - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,null,T,F), numDocs).scoreDocs; - assertEquals("smallest and up", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",null,maxRP,F,T), numDocs).scoreDocs; - assertEquals("biggest and down", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,null,F,F), numDocs).scoreDocs; - assertEquals("not smallest, but up", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",null,maxRP,F,F), numDocs).scoreDocs; - assertEquals("not biggest, but down", numDocs-1, result.length); - - // very small sets - - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,minRP,F,F), numDocs).scoreDocs; - assertEquals("min,min,F,F", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",maxRP,maxRP,F,F), numDocs).scoreDocs; - assertEquals("max,max,F,F", 0, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",minRP,minRP,T,T), numDocs).scoreDocs; - assertEquals("min,min,T,T", 1, result.length); - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",null,minRP,F,T), numDocs).scoreDocs; - assertEquals("nul,min,F,T", 1, result.length); - - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",maxRP,maxRP,T,T), numDocs).scoreDocs; - assertEquals("max,max,T,T", 1, result.length); - result = search.search(q,DocValuesRangeFilter.newStringRange("rand",maxRP,null,T,F), numDocs).scoreDocs; - assertEquals("max,nul,T,T", 1, result.length); - } - - @Test - public void testFieldCacheRangeFilterInts() throws IOException { - - IndexReader reader = signedIndexReader; - IndexSearcher search = newSearcher(reader); - - int numDocs = reader.numDocs(); - int medId = ((maxId - minId) / 2); - Integer minIdO = Integer.valueOf(minId); - Integer maxIdO = Integer.valueOf(maxId); - Integer medIdO = Integer.valueOf(medId); - - assertEquals("num of docs", numDocs, 1+ maxId - minId); - - ScoreDoc[] result; - Query q = new TermQuery(new Term("body","body")); - - // test id, bounded on both ends - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,maxIdO,T,T), numDocs).scoreDocs; - assertEquals("find all", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,maxIdO,T,F), numDocs).scoreDocs; - assertEquals("all but last", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,maxIdO,F,T), numDocs).scoreDocs; - assertEquals("all but first", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,maxIdO,F,F), numDocs).scoreDocs; - assertEquals("all but ends", numDocs-2, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",medIdO,maxIdO,T,T), numDocs).scoreDocs; - assertEquals("med and up", 1+ maxId-medId, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,medIdO,T,T), numDocs).scoreDocs; - assertEquals("up to med", 1+ medId-minId, result.length); - - // unbounded id - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",null,null,T,T), numDocs).scoreDocs; - assertEquals("find all", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,null,T,F), numDocs).scoreDocs; - assertEquals("min and up", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",null,maxIdO,F,T), numDocs).scoreDocs; - assertEquals("max and down", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,null,F,F), numDocs).scoreDocs; - assertEquals("not min, but up", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",null,maxIdO,F,F), numDocs).scoreDocs; - assertEquals("not max, but down", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",medIdO,maxIdO,T,F), numDocs).scoreDocs; - assertEquals("med and up, not max", maxId-medId, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,medIdO,F,T), numDocs).scoreDocs; - assertEquals("not min, up to med", medId-minId, result.length); - - // very small sets - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,minIdO,F,F), numDocs).scoreDocs; - assertEquals("min,min,F,F", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",medIdO,medIdO,F,F), numDocs).scoreDocs; - assertEquals("med,med,F,F", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",maxIdO,maxIdO,F,F), numDocs).scoreDocs; - assertEquals("max,max,F,F", 0, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",minIdO,minIdO,T,T), numDocs).scoreDocs; - assertEquals("min,min,T,T", 1, result.length); - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",null,minIdO,F,T), numDocs).scoreDocs; - assertEquals("nul,min,F,T", 1, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",maxIdO,maxIdO,T,T), numDocs).scoreDocs; - assertEquals("max,max,T,T", 1, result.length); - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",maxIdO,null,T,F), numDocs).scoreDocs; - assertEquals("max,nul,T,T", 1, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",medIdO,medIdO,T,T), numDocs).scoreDocs; - assertEquals("med,med,T,T", 1, result.length); - - // special cases - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",Integer.valueOf(Integer.MAX_VALUE),null,F,F), numDocs).scoreDocs; - assertEquals("overflow special case", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",null,Integer.valueOf(Integer.MIN_VALUE),F,F), numDocs).scoreDocs; - assertEquals("overflow special case", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",maxIdO,minIdO,T,T), numDocs).scoreDocs; - assertEquals("inverse range", 0, result.length); - } - - @Test - public void testFieldCacheRangeFilterLongs() throws IOException { - - IndexReader reader = signedIndexReader; - IndexSearcher search = newSearcher(reader); - - int numDocs = reader.numDocs(); - int medId = ((maxId - minId) / 2); - Long minIdO = Long.valueOf(minId); - Long maxIdO = Long.valueOf(maxId); - Long medIdO = Long.valueOf(medId); - - assertEquals("num of docs", numDocs, 1+ maxId - minId); - - ScoreDoc[] result; - Query q = new TermQuery(new Term("body","body")); - - // test id, bounded on both ends - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,maxIdO,T,T), numDocs).scoreDocs; - assertEquals("find all", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,maxIdO,T,F), numDocs).scoreDocs; - assertEquals("all but last", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,maxIdO,F,T), numDocs).scoreDocs; - assertEquals("all but first", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,maxIdO,F,F), numDocs).scoreDocs; - assertEquals("all but ends", numDocs-2, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",medIdO,maxIdO,T,T), numDocs).scoreDocs; - assertEquals("med and up", 1+ maxId-medId, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,medIdO,T,T), numDocs).scoreDocs; - assertEquals("up to med", 1+ medId-minId, result.length); - - // unbounded id - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",null,null,T,T), numDocs).scoreDocs; - assertEquals("find all", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,null,T,F), numDocs).scoreDocs; - assertEquals("min and up", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",null,maxIdO,F,T), numDocs).scoreDocs; - assertEquals("max and down", numDocs, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,null,F,F), numDocs).scoreDocs; - assertEquals("not min, but up", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",null,maxIdO,F,F), numDocs).scoreDocs; - assertEquals("not max, but down", numDocs-1, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",medIdO,maxIdO,T,F), numDocs).scoreDocs; - assertEquals("med and up, not max", maxId-medId, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,medIdO,F,T), numDocs).scoreDocs; - assertEquals("not min, up to med", medId-minId, result.length); - - // very small sets - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,minIdO,F,F), numDocs).scoreDocs; - assertEquals("min,min,F,F", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",medIdO,medIdO,F,F), numDocs).scoreDocs; - assertEquals("med,med,F,F", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",maxIdO,maxIdO,F,F), numDocs).scoreDocs; - assertEquals("max,max,F,F", 0, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",minIdO,minIdO,T,T), numDocs).scoreDocs; - assertEquals("min,min,T,T", 1, result.length); - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",null,minIdO,F,T), numDocs).scoreDocs; - assertEquals("nul,min,F,T", 1, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",maxIdO,maxIdO,T,T), numDocs).scoreDocs; - assertEquals("max,max,T,T", 1, result.length); - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",maxIdO,null,T,F), numDocs).scoreDocs; - assertEquals("max,nul,T,T", 1, result.length); - - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",medIdO,medIdO,T,T), numDocs).scoreDocs; - assertEquals("med,med,T,T", 1, result.length); - - // special cases - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",Long.valueOf(Long.MAX_VALUE),null,F,F), numDocs).scoreDocs; - assertEquals("overflow special case", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",null,Long.valueOf(Long.MIN_VALUE),F,F), numDocs).scoreDocs; - assertEquals("overflow special case", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newLongRange("id_long",maxIdO,minIdO,T,T), numDocs).scoreDocs; - assertEquals("inverse range", 0, result.length); - } - - // float and double tests are a bit minimalistic, but it's complicated, because missing precision - - @Test - public void testFieldCacheRangeFilterFloats() throws IOException { - - IndexReader reader = signedIndexReader; - IndexSearcher search = newSearcher(reader); - - int numDocs = reader.numDocs(); - Float minIdO = Float.valueOf(minId + .5f); - Float medIdO = Float.valueOf(minIdO.floatValue() + ((maxId-minId))/2.0f); - - ScoreDoc[] result; - Query q = new TermQuery(new Term("body","body")); - - result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",minIdO,medIdO,T,T), numDocs).scoreDocs; - assertEquals("find all", numDocs/2, result.length); - int count = 0; - result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",null,medIdO,F,T), numDocs).scoreDocs; - count += result.length; - result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",medIdO,null,F,F), numDocs).scoreDocs; - count += result.length; - assertEquals("sum of two concenatted ranges", numDocs, count); - result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",null,null,T,T), numDocs).scoreDocs; - assertEquals("find all", numDocs, result.length); - result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",Float.valueOf(Float.POSITIVE_INFINITY),null,F,F), numDocs).scoreDocs; - assertEquals("infinity special case", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newFloatRange("id_float",null,Float.valueOf(Float.NEGATIVE_INFINITY),F,F), numDocs).scoreDocs; - assertEquals("infinity special case", 0, result.length); - } - - @Test - public void testFieldCacheRangeFilterDoubles() throws IOException { - - IndexReader reader = signedIndexReader; - IndexSearcher search = newSearcher(reader); - - int numDocs = reader.numDocs(); - Double minIdO = Double.valueOf(minId + .5); - Double medIdO = Double.valueOf(minIdO.floatValue() + ((maxId-minId))/2.0); - - ScoreDoc[] result; - Query q = new TermQuery(new Term("body","body")); - - result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",minIdO,medIdO,T,T), numDocs).scoreDocs; - assertEquals("find all", numDocs/2, result.length); - int count = 0; - result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",null,medIdO,F,T), numDocs).scoreDocs; - count += result.length; - result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",medIdO,null,F,F), numDocs).scoreDocs; - count += result.length; - assertEquals("sum of two concenatted ranges", numDocs, count); - result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",null,null,T,T), numDocs).scoreDocs; - assertEquals("find all", numDocs, result.length); - result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",Double.valueOf(Double.POSITIVE_INFINITY),null,F,F), numDocs).scoreDocs; - assertEquals("infinity special case", 0, result.length); - result = search.search(q,DocValuesRangeFilter.newDoubleRange("id_double",null, Double.valueOf(Double.NEGATIVE_INFINITY),F,F), numDocs).scoreDocs; - assertEquals("infinity special case", 0, result.length); - } - - // test using a sparse index (with deleted docs). - @Test - public void testSparseIndex() throws IOException { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); - - for (int d = -20; d <= 20; d++) { - Document doc = new Document(); - doc.add(new IntField("id_int", d, Field.Store.NO)); - doc.add(new NumericDocValuesField("id_int", d)); - doc.add(newStringField("body", "body", Field.Store.NO)); - writer.addDocument(doc); - } - - writer.forceMerge(1); - BytesRefBuilder term0 = new BytesRefBuilder(); - NumericUtils.intToPrefixCoded(0, 0, term0); - writer.deleteDocuments(new Term("id_int", term0.get())); - writer.close(); - - IndexReader reader = DirectoryReader.open(dir); - IndexSearcher search = newSearcher(reader); - assertTrue(reader.hasDeletions()); - - ScoreDoc[] result; - Query q = new TermQuery(new Term("body","body")); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",-20,20,T,T), 100).scoreDocs; - assertEquals("find all", 40, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",0,20,T,T), 100).scoreDocs; - assertEquals("find all", 20, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",-20,0,T,T), 100).scoreDocs; - assertEquals("find all", 20, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",10,20,T,T), 100).scoreDocs; - assertEquals("find all", 11, result.length); - - result = search.search(q,DocValuesRangeFilter.newIntRange("id_int",-20,-10,T,T), 100).scoreDocs; - assertEquals("find all", 11, result.length); - reader.close(); - dir.close(); - } - -} diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFieldValueFilter.java b/lucene/core/src/test/org/apache/lucene/search/TestFieldValueFilter.java deleted file mode 100644 index 9703c06c3db..00000000000 --- a/lucene/core/src/test/org/apache/lucene/search/TestFieldValueFilter.java +++ /dev/null @@ -1,170 +0,0 @@ -package org.apache.lucene.search; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.SortedDocValuesField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.FilterLeafReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BitDocIdSet; -import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.FixedBitSet; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.SparseFixedBitSet; - -/** - * - */ -public class TestFieldValueFilter extends LuceneTestCase { - - public void testFieldValueFilterNoValue() throws IOException { - Directory directory = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, - newIndexWriterConfig(new MockAnalyzer(random()))); - int docs = atLeast(10); - int[] docStates = buildIndex(writer, docs); - int numDocsNoValue = 0; - for (int i = 0; i < docStates.length; i++) { - if (docStates[i] == 0) { - numDocsNoValue++; - } - } - - IndexReader reader = DirectoryReader.open(directory); - IndexSearcher searcher = newSearcher(reader); - TopDocs search = searcher.search(new TermQuery(new Term("all", "test")), - new FieldValueFilter("some", true), docs); - assertEquals(search.totalHits, numDocsNoValue); - - ScoreDoc[] scoreDocs = search.scoreDocs; - for (ScoreDoc scoreDoc : scoreDocs) { - assertNull(reader.document(scoreDoc.doc).get("some")); - } - - reader.close(); - directory.close(); - } - - public void testFieldValueFilter() throws IOException { - Directory directory = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, - newIndexWriterConfig(new MockAnalyzer(random()))); - int docs = atLeast(10); - int[] docStates = buildIndex(writer, docs); - int numDocsWithValue = 0; - for (int i = 0; i < docStates.length; i++) { - if (docStates[i] == 1) { - numDocsWithValue++; - } - } - IndexReader reader = DirectoryReader.open(directory); - IndexSearcher searcher = newSearcher(reader); - Filter filter = new FieldValueFilter("some"); - TopDocs search = searcher.search(new TermQuery(new Term("all", "test")), filter, docs); - assertEquals(search.totalHits, numDocsWithValue); - - ScoreDoc[] scoreDocs = search.scoreDocs; - for (ScoreDoc scoreDoc : scoreDocs) { - assertEquals("value", reader.document(scoreDoc.doc).get("some")); - } - - reader.close(); - directory.close(); - } - - public void testOptimizations() throws IOException { - Directory directory = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), directory, - newIndexWriterConfig(new MockAnalyzer(random()))); - final int docs = atLeast(10); - buildIndex(writer, docs); - IndexReader reader = DirectoryReader.open(directory); - LeafReader leafReader = reader.leaves().get(0).reader(); - - FilterLeafReader filterReader = new FilterLeafReader(leafReader) { - @Override - public Bits getDocsWithField(String field) throws IOException { - switch (field) { - case "with_matchall": - return new Bits.MatchAllBits(maxDoc()); - case "with_matchno": - return new Bits.MatchNoBits(maxDoc()); - case "with_bitset": - BitSet b = random().nextBoolean() ? new SparseFixedBitSet(maxDoc()) : new FixedBitSet(maxDoc()); - b.set(random().nextInt(maxDoc())); - return b; - } - return super.getDocsWithField(field); - } - }; - - Filter filter = new FieldValueFilter("with_matchall", true); - DocIdSet set = filter.getDocIdSet(filterReader.getContext(), null); - assertNull(set); - - filter = new FieldValueFilter("with_matchno"); - set = filter.getDocIdSet(filterReader.getContext(), null); - assertNull(set); - - filter = new FieldValueFilter("with_bitset"); - set = filter.getDocIdSet(filterReader.getContext(), null); - assertTrue(set instanceof BitDocIdSet); - - reader.close(); - directory.close(); - } - - private int[] buildIndex(RandomIndexWriter writer, int docs) - throws IOException { - int[] docStates = new int[docs]; - for (int i = 0; i < docs; i++) { - Document doc = new Document(); - if (random().nextBoolean()) { - docStates[i] = 1; - doc.add(newTextField("some", "value", Field.Store.YES)); - doc.add(new SortedDocValuesField("some", new BytesRef("value"))); - } - doc.add(newTextField("all", "test", Field.Store.NO)); - doc.add(new SortedDocValuesField("all", new BytesRef("test"))); - doc.add(newTextField("id", "" + i, Field.Store.YES)); - doc.add(new SortedDocValuesField("id", new BytesRef("" + i))); - writer.addDocument(doc); - } - writer.commit(); - int numDeletes = random().nextInt(docs); - for (int i = 0; i < numDeletes; i++) { - int docID = random().nextInt(docs); - writer.deleteDocuments(new Term("id", "" + docID)); - docStates[docID] = 2; - } - writer.close(); - return docStates; - } - -} diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFieldValueQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestFieldValueQuery.java new file mode 100644 index 00000000000..e00b16b57fe --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/search/TestFieldValueQuery.java @@ -0,0 +1,216 @@ +package org.apache.lucene.search; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; + +public class TestFieldValueQuery extends LuceneTestCase { + + public void testRandom() throws IOException { + final int iters = atLeast(10); + for (int iter = 0; iter < iters; ++iter) { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + final int numDocs = atLeast(100); + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + final boolean hasValue = random().nextBoolean(); + if (hasValue) { + doc.add(new NumericDocValuesField("dv1", 1)); + doc.add(new SortedNumericDocValuesField("dv2", 1)); + doc.add(new SortedNumericDocValuesField("dv2", 2)); + doc.add(new StringField("has_value", "yes", Store.NO)); + } + doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO)); + iw.addDocument(doc); + } + if (random().nextBoolean()) { + iw.deleteDocuments(new TermQuery(new Term("f", "no"))); + } + iw.commit(); + final IndexReader reader = iw.getReader(); + final IndexSearcher searcher = newSearcher(reader); + iw.close(); + + assertSameMatches(searcher, new TermQuery(new Term("has_value", "yes")), new FieldValueQuery("dv1"), false); + assertSameMatches(searcher, new TermQuery(new Term("has_value", "yes")), new FieldValueQuery("dv2"), false); + + reader.close(); + dir.close(); + } + } + + public void testApproximation() throws IOException { + final int iters = atLeast(10); + for (int iter = 0; iter < iters; ++iter) { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + final int numDocs = atLeast(100); + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + final boolean hasValue = random().nextBoolean(); + if (hasValue) { + doc.add(new NumericDocValuesField("dv1", 1)); + doc.add(new SortedNumericDocValuesField("dv2", 1)); + doc.add(new SortedNumericDocValuesField("dv2", 2)); + doc.add(new StringField("has_value", "yes", Store.NO)); + } + doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO)); + iw.addDocument(doc); + } + if (random().nextBoolean()) { + iw.deleteDocuments(new TermQuery(new Term("f", "no"))); + } + iw.commit(); + final IndexReader reader = iw.getReader(); + final IndexSearcher searcher = newSearcher(reader); + iw.close(); + + BooleanQuery ref = new BooleanQuery(); + ref.add(new TermQuery(new Term("f", "yes")), Occur.MUST); + ref.add(new TermQuery(new Term("has_value", "yes")), Occur.FILTER); + + BooleanQuery bq1 = new BooleanQuery(); + bq1.add(new TermQuery(new Term("f", "yes")), Occur.MUST); + bq1.add(new FieldValueQuery("dv1"), Occur.FILTER); + assertSameMatches(searcher, ref, bq1, true); + + BooleanQuery bq2 = new BooleanQuery(); + bq2.add(new TermQuery(new Term("f", "yes")), Occur.MUST); + bq2.add(new FieldValueQuery("dv2"), Occur.FILTER); + assertSameMatches(searcher, ref, bq2, true); + + reader.close(); + dir.close(); + } + } + + public void testScore() throws IOException { + final int iters = atLeast(10); + for (int iter = 0; iter < iters; ++iter) { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + final int numDocs = atLeast(100); + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + final boolean hasValue = random().nextBoolean(); + if (hasValue) { + doc.add(new NumericDocValuesField("dv1", 1)); + doc.add(new SortedNumericDocValuesField("dv2", 1)); + doc.add(new SortedNumericDocValuesField("dv2", 2)); + doc.add(new StringField("has_value", "yes", Store.NO)); + } + doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO)); + iw.addDocument(doc); + } + if (random().nextBoolean()) { + iw.deleteDocuments(new TermQuery(new Term("f", "no"))); + } + iw.commit(); + final IndexReader reader = iw.getReader(); + final IndexSearcher searcher = newSearcher(reader); + iw.close(); + + final float boost = random().nextFloat() * 10; + final Query ref = new ConstantScoreQuery(new TermQuery(new Term("has_value", "yes"))); + ref.setBoost(boost); + + final Query q1 = new FieldValueQuery("dv1"); + q1.setBoost(boost); + assertSameMatches(searcher, ref, q1, true); + + final Query q2 = new FieldValueQuery("dv2"); + q2.setBoost(boost); + assertSameMatches(searcher, ref, q2, true); + + reader.close(); + dir.close(); + } + } + + public void testMissingField() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + iw.addDocument(new Document()); + iw.commit(); + final IndexReader reader = iw.getReader(); + final IndexSearcher searcher = newSearcher(reader); + iw.close(); + assertEquals(0, searcher.search(new FieldValueQuery("f"), 1).totalHits); + reader.close(); + dir.close(); + } + + public void testAllDocsHaveField() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new NumericDocValuesField("f", 1)); + iw.addDocument(doc); + iw.commit(); + final IndexReader reader = iw.getReader(); + final IndexSearcher searcher = newSearcher(reader); + iw.close(); + assertEquals(1, searcher.search(new FieldValueQuery("f"), 1).totalHits); + reader.close(); + dir.close(); + } + + public void testFieldExistsButNoDocsHaveField() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + // 1st segment has the field, but 2nd one does not + Document doc = new Document(); + doc.add(new NumericDocValuesField("f", 1)); + iw.addDocument(doc); + iw.commit(); + iw.addDocument(new Document()); + iw.commit(); + final IndexReader reader = iw.getReader(); + final IndexSearcher searcher = newSearcher(reader); + iw.close(); + assertEquals(1, searcher.search(new FieldValueQuery("f"), 1).totalHits); + reader.close(); + dir.close(); + } + + private void assertSameMatches(IndexSearcher searcher, Query q1, Query q2, boolean scores) throws IOException { + final int maxDoc = searcher.getIndexReader().maxDoc(); + final TopDocs td1 = searcher.search(q1, maxDoc, scores ? Sort.RELEVANCE : Sort.INDEXORDER); + final TopDocs td2 = searcher.search(q2, maxDoc, scores ? Sort.RELEVANCE : Sort.INDEXORDER); + assertEquals(td1.totalHits, td2.totalHits); + for (int i = 0; i < td1.scoreDocs.length; ++i) { + assertEquals(td1.scoreDocs[i].doc, td2.scoreDocs[i].doc); + if (scores) { + assertEquals(td1.scoreDocs[i].score, td2.scoreDocs[i].score, 10e-7); + } + } + } +} diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java index ecf4c55d68d..7aaa5e02711 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java @@ -19,9 +19,10 @@ package org.apache.lucene.sandbox.queries; import java.text.Collator; +import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.MultiTermQueryWrapperFilter; import org.apache.lucene.search.NumericRangeFilter; // javadoc -import org.apache.lucene.search.DocValuesRangeFilter; // javadoc +// javadoc /** * A Filter that restricts search results to a range of term @@ -33,7 +34,7 @@ import org.apache.lucene.search.DocValuesRangeFilter; // javadoc * for numerical ranges; use {@link NumericRangeFilter} instead. * *

If you construct a large number of range filters with different ranges but on the - * same field, {@link DocValuesRangeFilter} may have significantly better performance. + * same field, {@link DocValuesRangeQuery} may have significantly better performance. * @deprecated Index collation keys with CollationKeyAnalyzer or ICUCollationKeyAnalyzer instead. * This class will be removed in Lucene 5.0 */ diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java index 83be3170881..d167251b329 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java @@ -28,20 +28,17 @@ import org.apache.commons.io.IOUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; +import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.collation.ICUCollationKeyAnalyzer; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.StorableField; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.DocTermOrdsRangeFilter; -import org.apache.lucene.search.DocValuesRangeFilter; +import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.uninverting.UninvertingReader.Type; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.Version; -import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.response.TextResponseWriter; @@ -277,11 +274,11 @@ public class ICUCollationField extends FieldType { BytesRef high = part2 == null ? null : getCollationKey(f, part2); if (!field.indexed() && field.hasDocValues()) { if (field.multiValued()) { - return new ConstantScoreQuery(DocTermOrdsRangeFilter.newBytesRefRange( - field.getName(), low, high, minInclusive, maxInclusive)); + return DocValuesRangeQuery.newBytesRefRange( + field.getName(), low, high, minInclusive, maxInclusive); } else { - return new ConstantScoreQuery(DocValuesRangeFilter.newBytesRefRange( - field.getName(), low, high, minInclusive, maxInclusive)); + return DocValuesRangeQuery.newBytesRefRange( + field.getName(), low, high, minInclusive, maxInclusive); } } else { return new TermRangeQuery(field.getName(), low, high, minInclusive, maxInclusive); diff --git a/solr/core/src/java/org/apache/solr/schema/CollationField.java b/solr/core/src/java/org/apache/solr/schema/CollationField.java index aa25481aae2..fc0216e61ae 100644 --- a/solr/core/src/java/org/apache/solr/schema/CollationField.java +++ b/solr/core/src/java/org/apache/solr/schema/CollationField.java @@ -32,20 +32,17 @@ import org.apache.commons.io.IOUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; +import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.collation.CollationKeyAnalyzer; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.StorableField; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.DocTermOrdsRangeFilter; -import org.apache.lucene.search.DocValuesRangeFilter; +import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.uninverting.UninvertingReader.Type; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.Version; -import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.response.TextResponseWriter; @@ -248,13 +245,8 @@ public class CollationField extends FieldType { BytesRef low = part1 == null ? null : getCollationKey(f, part1); BytesRef high = part2 == null ? null : getCollationKey(f, part2); if (!field.indexed() && field.hasDocValues()) { - if (field.multiValued()) { - return new ConstantScoreQuery(DocTermOrdsRangeFilter.newBytesRefRange( - field.getName(), low, high, minInclusive, maxInclusive)); - } else { - return new ConstantScoreQuery(DocValuesRangeFilter.newBytesRefRange( - field.getName(), low, high, minInclusive, maxInclusive)); - } + return DocValuesRangeQuery.newBytesRefRange( + field.getName(), low, high, minInclusive, maxInclusive); } else { return new TermRangeQuery(field.getName(), low, high, minInclusive, maxInclusive); } diff --git a/solr/core/src/java/org/apache/solr/schema/CurrencyField.java b/solr/core/src/java/org/apache/solr/schema/CurrencyField.java index 2170607d932..3adf119e33b 100644 --- a/solr/core/src/java/org/apache/solr/schema/CurrencyField.java +++ b/solr/core/src/java/org/apache/solr/schema/CurrencyField.java @@ -16,19 +16,37 @@ package org.apache.solr.schema; * limitations under the License. */ +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Currency; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; + import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.analysis.util.ResourceLoaderAware; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.StorableField; +import org.apache.lucene.queries.BooleanFilter; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.SortField; +import org.apache.lucene.search.FieldValueQuery; import org.apache.lucene.search.Filter; -import org.apache.lucene.search.FieldValueFilter; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.SortField; import org.apache.lucene.uninverting.UninvertingReader.Type; -import org.apache.lucene.queries.BooleanFilter; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.response.TextResponseWriter; @@ -43,23 +61,6 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Currency; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - /** * Field type for support of monetary values. *

@@ -328,7 +329,7 @@ public class CurrencyField extends FieldType implements SchemaAware, ResourceLoa (p2 != null) ? p2.getCurrencyCode() : defaultCurrency; // ValueSourceRangeFilter doesn't check exists(), so we have to - final Filter docsWithValues = new FieldValueFilter(getAmountField(field).getName()); + final Filter docsWithValues = new QueryWrapperFilter(new FieldValueQuery(getAmountField(field).getName())); final Filter vsRangeFilter = new ValueSourceRangeFilter (new RawCurrencyValueSource(field, currencyCode, parser), p1 == null ? null : p1.getAmount() + "", diff --git a/solr/core/src/java/org/apache/solr/schema/EnumField.java b/solr/core/src/java/org/apache/solr/schema/EnumField.java index 24b43122ec5..bea245a8587 100644 --- a/solr/core/src/java/org/apache/solr/schema/EnumField.java +++ b/solr/core/src/java/org/apache/solr/schema/EnumField.java @@ -237,9 +237,9 @@ public class EnumField extends PrimitiveFieldType { Query query = null; final boolean matchOnly = field.hasDocValues() && !field.indexed(); if (matchOnly) { - query = new ConstantScoreQuery(DocValuesRangeFilter.newIntRange(field.getName(), - min == null ? null : minValue, - max == null ? null : maxValue, + query = new ConstantScoreQuery(DocValuesRangeQuery.newLongRange(field.getName(), + min == null ? null : minValue.longValue(), + max == null ? null : maxValue.longValue(), minInclusive, maxInclusive)); } else { query = NumericRangeQuery.newIntRange(field.getName(), DEFAULT_PRECISION_STEP, diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 6ac68c02d87..32a45dd060f 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -17,6 +17,17 @@ package org.apache.solr.schema; +import static org.apache.lucene.analysis.util.AbstractAnalysisFactory.LUCENE_MATCH_VERSION_PARAM; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -29,10 +40,8 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.StorableField; import org.apache.lucene.index.Term; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.DocTermOrdsRangeFilter; import org.apache.lucene.search.DocTermOrdsRewriteMethod; -import org.apache.lucene.search.DocValuesRangeFilter; +import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.DocValuesRewriteMethod; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; @@ -46,7 +55,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.CharsRefBuilder; -import org.apache.lucene.util.UnicodeUtil; import org.apache.solr.analysis.SolrAnalyzer; import org.apache.solr.analysis.TokenizerChain; import org.apache.solr.common.SolrException; @@ -60,17 +68,6 @@ import org.apache.solr.search.Sorting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.apache.lucene.analysis.util.AbstractAnalysisFactory.LUCENE_MATCH_VERSION_PARAM; - /** * Base class for all field types used by an index schema. * @@ -692,19 +689,11 @@ public abstract class FieldType extends FieldProperties { public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) { // TODO: change these all to use readableToIndexed/bytes instead (e.g. for unicode collation) if (field.hasDocValues() && !field.indexed()) { - if (field.multiValued()) { - return new ConstantScoreQuery(DocTermOrdsRangeFilter.newBytesRefRange( - field.getName(), - part1 == null ? null : new BytesRef(toInternal(part1)), - part2 == null ? null : new BytesRef(toInternal(part2)), - minInclusive, maxInclusive)); - } else { - return new ConstantScoreQuery(DocValuesRangeFilter.newStringRange( - field.getName(), - part1 == null ? null : toInternal(part1), - part2 == null ? null : toInternal(part2), - minInclusive, maxInclusive)); - } + return DocValuesRangeQuery.newBytesRefRange( + field.getName(), + part1 == null ? null : new BytesRef(toInternal(part1)), + part2 == null ? null : new BytesRef(toInternal(part2)), + minInclusive, maxInclusive); } else { MultiTermQuery rangeQuery = TermRangeQuery.newStringRange( field.getName(), diff --git a/solr/core/src/java/org/apache/solr/schema/TrieField.java b/solr/core/src/java/org/apache/solr/schema/TrieField.java index ccf181933fe..84d526022b8 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieField.java @@ -39,8 +39,7 @@ import org.apache.lucene.queries.function.valuesource.DoubleFieldSource; import org.apache.lucene.queries.function.valuesource.FloatFieldSource; import org.apache.lucene.queries.function.valuesource.IntFieldSource; import org.apache.lucene.queries.function.valuesource.LongFieldSource; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.DocValuesRangeFilter; +import org.apache.lucene.search.DocValuesRangeQuery; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; @@ -310,10 +309,10 @@ public class TrieField extends PrimitiveFieldType { switch (type) { case INTEGER: if (matchOnly) { - query = new ConstantScoreQuery(DocValuesRangeFilter.newIntRange(field.getName(), - min == null ? null : Integer.parseInt(min), - max == null ? null : Integer.parseInt(max), - minInclusive, maxInclusive)); + query = DocValuesRangeQuery.newLongRange(field.getName(), + min == null ? null : (long) Integer.parseInt(min), + max == null ? null : (long) Integer.parseInt(max), + minInclusive, maxInclusive); } else { query = NumericRangeQuery.newIntRange(field.getName(), ps, min == null ? null : Integer.parseInt(min), @@ -323,10 +322,10 @@ public class TrieField extends PrimitiveFieldType { break; case FLOAT: if (matchOnly) { - query = new ConstantScoreQuery(DocValuesRangeFilter.newFloatRange(field.getName(), - min == null ? null : Float.parseFloat(min), - max == null ? null : Float.parseFloat(max), - minInclusive, maxInclusive)); + query = DocValuesRangeQuery.newLongRange(field.getName(), + min == null ? null : (long) NumericUtils.floatToSortableInt(Float.parseFloat(min)), + max == null ? null : (long) NumericUtils.floatToSortableInt(Float.parseFloat(max)), + minInclusive, maxInclusive); } else { query = NumericRangeQuery.newFloatRange(field.getName(), ps, min == null ? null : Float.parseFloat(min), @@ -336,10 +335,10 @@ public class TrieField extends PrimitiveFieldType { break; case LONG: if (matchOnly) { - query = new ConstantScoreQuery(DocValuesRangeFilter.newLongRange(field.getName(), + query = DocValuesRangeQuery.newLongRange(field.getName(), min == null ? null : Long.parseLong(min), max == null ? null : Long.parseLong(max), - minInclusive, maxInclusive)); + minInclusive, maxInclusive); } else { query = NumericRangeQuery.newLongRange(field.getName(), ps, min == null ? null : Long.parseLong(min), @@ -349,10 +348,10 @@ public class TrieField extends PrimitiveFieldType { break; case DOUBLE: if (matchOnly) { - query = new ConstantScoreQuery(DocValuesRangeFilter.newDoubleRange(field.getName(), - min == null ? null : Double.parseDouble(min), - max == null ? null : Double.parseDouble(max), - minInclusive, maxInclusive)); + query = DocValuesRangeQuery.newLongRange(field.getName(), + min == null ? null : NumericUtils.doubleToSortableLong(Double.parseDouble(min)), + max == null ? null : NumericUtils.doubleToSortableLong(Double.parseDouble(max)), + minInclusive, maxInclusive); } else { query = NumericRangeQuery.newDoubleRange(field.getName(), ps, min == null ? null : Double.parseDouble(min), @@ -362,10 +361,10 @@ public class TrieField extends PrimitiveFieldType { break; case DATE: if (matchOnly) { - query = new ConstantScoreQuery(DocValuesRangeFilter.newLongRange(field.getName(), + query = DocValuesRangeQuery.newLongRange(field.getName(), min == null ? null : dateField.parseMath(null, min).getTime(), max == null ? null : dateField.parseMath(null, max).getTime(), - minInclusive, maxInclusive)); + minInclusive, maxInclusive); } else { query = NumericRangeQuery.newLongRange(field.getName(), ps, min == null ? null : dateField.parseMath(null, min).getTime(),