SOLR-11070, SOLR-11056: Make docValues range queries behave the same as Trie/Point fields for Double/Float Infinity cases

This commit is contained in:
Tomas Fernandez Lobbe 2017-07-26 10:10:40 -07:00
parent 67b3d4e108
commit 847ab9e326
7 changed files with 529 additions and 87 deletions

View File

@ -415,6 +415,9 @@ Optimizations
* SOLR-10727: Avoid polluting the filter cache for certain types of faceting (typically ranges) when
the base docset is empty. (David Smiley)
* SOLR-11070: Make docValues range queries behave the same as Trie/Point fields for Double/Float Infinity cases
(Tomás Fernández Löbbe, Andrey Kudryavtsev)
Other Changes
----------------------
* SOLR-10236: Removed FieldType.getNumericType(). Use getNumberType() instead. (Tomás Fernández Löbbe)
@ -559,6 +562,9 @@ Other Changes
* SOLR-10760: Remove trie field types and fields from example schemas. (Steve Rowe)
* SOLR-11056: Add random range query test that compares results across Trie*, *Point and DocValue-only fields
(Tomás Fernández Löbbe)
================== 6.7.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

View File

@ -18,7 +18,6 @@
package org.apache.solr.schema;
import java.util.Collection;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DocValuesType;
@ -26,6 +25,7 @@ import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleFieldSource;
import org.apache.lucene.queries.function.valuesource.MultiValuedDoubleFieldSource;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedNumericSelector;
@ -63,6 +63,7 @@ public class DoublePointField extends PointField implements DoubleValueFieldType
} else {
actualMin = parseDoubleFromUser(field.getName(), min);
if (!minInclusive) {
if (actualMin == Double.POSITIVE_INFINITY) return new MatchNoDocsQuery();
actualMin = DoublePoint.nextUp(actualMin);
}
}
@ -71,6 +72,7 @@ public class DoublePointField extends PointField implements DoubleValueFieldType
} else {
actualMax = parseDoubleFromUser(field.getName(), max);
if (!maxInclusive) {
if (actualMax == Double.NEGATIVE_INFINITY) return new MatchNoDocsQuery();
actualMax = DoublePoint.nextDown(actualMax);
}
}

View File

@ -18,7 +18,6 @@
package org.apache.solr.schema;
import java.util.Collection;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DocValuesType;
@ -26,6 +25,7 @@ import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.MultiValuedFloatFieldSource;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedNumericSelector;
@ -63,6 +63,7 @@ public class FloatPointField extends PointField implements FloatValueFieldType {
} else {
actualMin = parseFloatFromUser(field.getName(), min);
if (!minInclusive) {
if (actualMin == Float.POSITIVE_INFINITY) return new MatchNoDocsQuery();
actualMin = FloatPoint.nextUp(actualMin);
}
}
@ -71,6 +72,7 @@ public class FloatPointField extends PointField implements FloatValueFieldType {
} else {
actualMax = parseFloatFromUser(field.getName(), max);
if (!maxInclusive) {
if (actualMax == Float.NEGATIVE_INFINITY) return new MatchNoDocsQuery();
actualMax = FloatPoint.nextDown(actualMax);
}
}

View File

@ -16,6 +16,8 @@
*/
package org.apache.solr.schema;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.queries.function.ValueSource;
@ -40,14 +42,8 @@ public abstract class NumericFieldType extends PrimitiveFieldType {
return type;
}
private static long FLOAT_NEGATIVE_INFINITY_BITS = (long)Float.floatToIntBits(Float.NEGATIVE_INFINITY);
private static long DOUBLE_NEGATIVE_INFINITY_BITS = Double.doubleToLongBits(Double.NEGATIVE_INFINITY);
private static long FLOAT_POSITIVE_INFINITY_BITS = (long)Float.floatToIntBits(Float.POSITIVE_INFINITY);
private static long DOUBLE_POSITIVE_INFINITY_BITS = Double.doubleToLongBits(Double.POSITIVE_INFINITY);
private static long FLOAT_MINUS_ZERO_BITS = (long)Float.floatToIntBits(-0f);
private static long DOUBLE_MINUS_ZERO_BITS = Double.doubleToLongBits(-0d);
private static long FLOAT_ZERO_BITS = (long)Float.floatToIntBits(0f);
private static long DOUBLE_ZERO_BITS = Double.doubleToLongBits(0d);
protected Query getDocValuesRangeQuery(QParser parser, SchemaField field, String min, String max,
boolean minInclusive, boolean maxInclusive) {
@ -89,50 +85,119 @@ public abstract class NumericFieldType extends PrimitiveFieldType {
protected Query getRangeQueryForFloatDoubleDocValues(SchemaField sf, String min, String max, boolean minInclusive, boolean maxInclusive) {
Query query;
String fieldName = sf.getName();
Number minVal = min == null ? null : getNumberType() == NumberType.FLOAT ? parseFloatFromUser(sf.getName(), min): parseDoubleFromUser(sf.getName(), min);
Number maxVal = max == null ? null : getNumberType() == NumberType.FLOAT ? parseFloatFromUser(sf.getName(), max): parseDoubleFromUser(sf.getName(), max);
Long minBits =
min == null ? null : getNumberType() == NumberType.FLOAT ? (long) Float.floatToIntBits(minVal.floatValue()): Double.doubleToLongBits(minVal.doubleValue());
Long maxBits =
max == null ? null : getNumberType() == NumberType.FLOAT ? (long) Float.floatToIntBits(maxVal.floatValue()): Double.doubleToLongBits(maxVal.doubleValue());
long negativeInfinityBits = getNumberType() == NumberType.FLOAT ? FLOAT_NEGATIVE_INFINITY_BITS : DOUBLE_NEGATIVE_INFINITY_BITS;
long positiveInfinityBits = getNumberType() == NumberType.FLOAT ? FLOAT_POSITIVE_INFINITY_BITS : DOUBLE_POSITIVE_INFINITY_BITS;
long minusZeroBits = getNumberType() == NumberType.FLOAT ? FLOAT_MINUS_ZERO_BITS : DOUBLE_MINUS_ZERO_BITS;
long zeroBits = getNumberType() == NumberType.FLOAT ? FLOAT_ZERO_BITS : DOUBLE_ZERO_BITS;
// If min is negative (or -0d) and max is positive (or +0d), then issue a FunctionRangeQuery
if ((minVal == null || minVal.doubleValue() < 0d || minBits == minusZeroBits) &&
(maxVal == null || (maxVal.doubleValue() > 0d || maxBits == zeroBits))) {
ValueSource vs = getValueSource(sf, null);
query = new FunctionRangeQuery(new ValueSourceRangeFilter(vs, min, max, minInclusive, maxInclusive));
} else { // If both max and min are negative (or -0d), then issue range query with max and min reversed
if ((minVal == null || minVal.doubleValue() < 0d || minBits == minusZeroBits) &&
(maxVal != null && (maxVal.doubleValue() < 0d || maxBits == minusZeroBits))) {
query = numericDocValuesRangeQuery
(fieldName, maxBits, (min == null ? Long.valueOf(negativeInfinityBits) : minBits), maxInclusive, minInclusive, false);
} else { // If both max and min are positive, then issue range query
query = numericDocValuesRangeQuery
(fieldName, minBits, (max == null ? Long.valueOf(positiveInfinityBits) : maxBits), minInclusive, maxInclusive, false);
long minBits, maxBits;
boolean minNegative, maxNegative;
Number minVal, maxVal;
if (getNumberType() == NumberType.FLOAT) {
if (min == null) {
minVal = Float.NEGATIVE_INFINITY;
} else {
minVal = parseFloatFromUser(sf.getName(), min);
if (!minInclusive) {
if (minVal.floatValue() == Float.POSITIVE_INFINITY) return new MatchNoDocsQuery();
minVal = FloatPoint.nextUp(minVal.floatValue());
}
}
if (max == null) {
maxVal = Float.POSITIVE_INFINITY;
} else {
maxVal = parseFloatFromUser(sf.getName(), max);
if (!maxInclusive) {
if (maxVal.floatValue() == Float.NEGATIVE_INFINITY) return new MatchNoDocsQuery();
maxVal = FloatPoint.nextDown(maxVal.floatValue());
}
}
minBits = Float.floatToIntBits(minVal.floatValue());
maxBits = Float.floatToIntBits(maxVal.floatValue());
minNegative = minVal.floatValue() < 0f || minBits == FLOAT_MINUS_ZERO_BITS;
maxNegative = maxVal.floatValue() < 0f || maxBits == FLOAT_MINUS_ZERO_BITS;
} else {
assert getNumberType() == NumberType.DOUBLE;
if (min == null) {
minVal = Double.NEGATIVE_INFINITY;
} else {
minVal = parseDoubleFromUser(sf.getName(), min);
if (!minInclusive) {
if (minVal.doubleValue() == Double.POSITIVE_INFINITY) return new MatchNoDocsQuery();
minVal = DoublePoint.nextUp(minVal.doubleValue());
}
}
if (max == null) {
maxVal = Double.POSITIVE_INFINITY;
} else {
maxVal = parseDoubleFromUser(sf.getName(), max);
if (!maxInclusive) {
if (maxVal.doubleValue() == Double.NEGATIVE_INFINITY) return new MatchNoDocsQuery();
maxVal = DoublePoint.nextDown(maxVal.doubleValue());
}
}
minBits = Double.doubleToLongBits(minVal.doubleValue());
maxBits = Double.doubleToLongBits(maxVal.doubleValue());
minNegative = minVal.doubleValue() < 0d || minBits == DOUBLE_MINUS_ZERO_BITS;
maxNegative = maxVal.doubleValue() < 0d || maxBits == DOUBLE_MINUS_ZERO_BITS;
}
// If min is negative (or -0d) and max is positive (or +0d), then issue a FunctionRangeQuery
if (minNegative && !maxNegative) {
ValueSource vs = getValueSource(sf, null);
query = new FunctionRangeQuery(new ValueSourceRangeFilter(vs, minVal.toString(), maxVal.toString(), true, true));
} else if (minNegative && maxNegative) {// If both max and min are negative (or -0d), then issue range query with max and min reversed
query = numericDocValuesRangeQuery
(fieldName, maxBits, minBits, true, true, false);
} else { // If both max and min are positive, then issue range query
query = numericDocValuesRangeQuery
(fieldName, minBits, maxBits, true, true, false);
}
return query;
}
protected Query getRangeQueryForMultiValuedDoubleDocValues(SchemaField sf, String min, String max, boolean minInclusive, boolean maxInclusive) {
Long minBits = min == null ? NumericUtils.doubleToSortableLong(Double.NEGATIVE_INFINITY): NumericUtils.doubleToSortableLong(parseDoubleFromUser(sf.getName(), min));
Long maxBits = max == null ? NumericUtils.doubleToSortableLong(Double.POSITIVE_INFINITY): NumericUtils.doubleToSortableLong(parseDoubleFromUser(sf.getName(), max));
return numericDocValuesRangeQuery(sf.getName(), minBits, maxBits, minInclusive, maxInclusive, true);
double minVal,maxVal;
if (min == null) {
minVal = Double.NEGATIVE_INFINITY;
} else {
minVal = parseDoubleFromUser(sf.getName(), min);
if (!minInclusive) {
if (minVal == Double.POSITIVE_INFINITY) return new MatchNoDocsQuery();
minVal = DoublePoint.nextUp(minVal);
}
}
if (max == null) {
maxVal = Double.POSITIVE_INFINITY;
} else {
maxVal = parseDoubleFromUser(sf.getName(), max);
if (!maxInclusive) {
if (maxVal == Double.NEGATIVE_INFINITY) return new MatchNoDocsQuery();
maxVal = DoublePoint.nextDown(maxVal);
}
}
Long minBits = NumericUtils.doubleToSortableLong(minVal);
Long maxBits = NumericUtils.doubleToSortableLong(maxVal);
return numericDocValuesRangeQuery(sf.getName(), minBits, maxBits, true, true, true);
}
protected Query getRangeQueryForMultiValuedFloatDocValues(SchemaField sf, String min, String max, boolean minInclusive, boolean maxInclusive) {
Long minBits = (long)(min == null ? NumericUtils.floatToSortableInt(Float.NEGATIVE_INFINITY): NumericUtils.floatToSortableInt(parseFloatFromUser(sf.getName(), min)));
Long maxBits = (long)(max == null ? NumericUtils.floatToSortableInt(Float.POSITIVE_INFINITY): NumericUtils.floatToSortableInt(parseFloatFromUser(sf.getName(), max)));
return numericDocValuesRangeQuery(sf.getName(), minBits, maxBits, minInclusive, maxInclusive, true);
float minVal,maxVal;
if (min == null) {
minVal = Float.NEGATIVE_INFINITY;
} else {
minVal = parseFloatFromUser(sf.getName(), min);
if (!minInclusive) {
if (minVal == Float.POSITIVE_INFINITY) return new MatchNoDocsQuery();
minVal = FloatPoint.nextUp(minVal);
}
}
if (max == null) {
maxVal = Float.POSITIVE_INFINITY;
} else {
maxVal = parseFloatFromUser(sf.getName(), max);
if (!maxInclusive) {
if (maxVal == Float.NEGATIVE_INFINITY) return new MatchNoDocsQuery();
maxVal = FloatPoint.nextDown(maxVal);
}
}
Long minBits = (long)NumericUtils.floatToSortableInt(minVal);
Long maxBits = (long)NumericUtils.floatToSortableInt(maxVal);
return numericDocValuesRangeQuery(sf.getName(), minBits, maxBits, true, true, true);
}
public static Query numericDocValuesRangeQuery(

View File

@ -412,7 +412,16 @@ valued. -->
<dynamicField name="*_dt_ni_p" type="pdate" indexed="false" stored="true" docValues="true"/>
<dynamicField name="*_dts_ni_p" type="pdate" indexed="false" stored="true" docValues="true" multiValued="true"/>
<dynamicField name="*_i_ndv_p" type="pint" indexed="true" stored="true" docValues="false" multiValued="false"/>
<dynamicField name="*_is_ndv_p" type="pint" indexed="true" stored="true" docValues="false" multiValued="true"/>
<dynamicField name="*_l_ndv_p" type="plong" indexed="true" stored="true" docValues="false" multiValued="false"/>
<dynamicField name="*_ls_ndv_p" type="plong" indexed="true" stored="true" docValues="false" multiValued="true"/>
<dynamicField name="*_f_ndv_p" type="pfloat" indexed="true" stored="true" docValues="false" multiValued="false"/>
<dynamicField name="*_fs_ndv_p" type="pfloat" indexed="true" stored="true" docValues="false" multiValued="true"/>
<dynamicField name="*_d_ndv_p" type="pdouble" indexed="true" stored="true" docValues="false" multiValued="false"/>
<dynamicField name="*_ds_ndv_p" type="pdouble" indexed="true" stored="true" docValues="false" multiValued="true"/>
<dynamicField name="*_dt_ndv_p" type="pdate" indexed="true" stored="true" docValues="false" multiValued="false"/>
<dynamicField name="*_dts_ndv_p" type="pdate" indexed="true" stored="true" docValues="false" multiValued="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>

View File

@ -2648,8 +2648,9 @@ public class TestPointFields extends SolrTestCaseJ4 {
private void doTestPointFieldMultiValuedRangeQuery(String fieldName, String type, String[] numbers) throws Exception {
assert numbers != null && numbers.length == 20;
assertTrue(h.getCore().getLatestSchema().getField(fieldName).multiValued());
assertTrue(h.getCore().getLatestSchema().getField(fieldName).getType() instanceof PointField);
SchemaField sf = h.getCore().getLatestSchema().getField(fieldName);
assertTrue(sf.multiValued());
assertTrue(sf.getType() instanceof PointField);
for (int i=9; i >= 0; i--) {
assertU(adoc("id", String.valueOf(i), fieldName, numbers[i], fieldName, numbers[i+10]));
}
@ -2723,6 +2724,11 @@ public class TestPointFields extends SolrTestCaseJ4 {
"fl", "id, " + fieldName, "sort", "id asc"),
"//*[@numFound='1']",
"//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[0] + "']");
if (sf.getType().getNumberType() == NumberType.FLOAT || sf.getType().getNumberType() == NumberType.DOUBLE) {
doTestDoubleFloatRangeLimits(fieldName, sf.getType().getNumberType() == NumberType.DOUBLE);
}
}
private void doTestPointFieldMultiValuedFacetField(String nonDocValuesField, String dvFieldName, String[] numbers) throws Exception {
@ -3191,45 +3197,98 @@ public class TestPointFields extends SolrTestCaseJ4 {
assertQ(req("q", fieldName + ":{" + arr[0] + " TO " + arr[i] + "}", "fl", "id, " + fieldName),
"//*[@numFound='" + (Math.max(0, i-1)) + "']");
}
clearIndex();
assertU(adoc("id", "1", fieldName, String.valueOf(Float.MAX_VALUE)));
assertU(adoc("id", "2", fieldName, String.valueOf(Float.MIN_VALUE)));
assertU(adoc("id", "3", fieldName, String.valueOf(Float.NEGATIVE_INFINITY)));
assertU(adoc("id", "4", fieldName, String.valueOf(Float.POSITIVE_INFINITY)));
assertU(commit());
assertQ(req("q", fieldName + ":[* TO *]", "fl", "id, " + fieldName),
"//*[@numFound='4']");
// TODO: Awaits fix: SOLR-11070
// assertQ(req("q", fieldName + ":{* TO *}", "fl", "id, " + fieldName),
// "//*[@numFound='4']");
assertQ(req("q", fieldName + ":[" + Float.MIN_VALUE + " TO " + Float.MAX_VALUE + "]", "fl", "id, " + fieldName),
"//*[@numFound='2']");
assertQ(req("q", fieldName + ":{" + Float.MIN_VALUE + " TO " + Float.MAX_VALUE + "]", "fl", "id, " + fieldName),
"//*[@numFound='1']");
assertQ(req("q", fieldName + ":[" + Float.MIN_VALUE + " TO " + Float.MAX_VALUE + "}", "fl", "id, " + fieldName),
"//*[@numFound='1']");
if (testDouble) {
assertQ(req("q", fieldName + ":[" + Double.MIN_VALUE + " TO " + Double.MIN_VALUE + "}", "fl", "id, " + fieldName),
"//*[@numFound='0']");
assertQ(req("q", fieldName + ":{" + Double.MAX_VALUE + " TO " + Double.MAX_VALUE + "]", "fl", "id, " + fieldName),
"//*[@numFound='0']");
assertQ(req("q", fieldName + ":{" + Double.NEGATIVE_INFINITY + " TO " + Double.NEGATIVE_INFINITY + "]", "fl", "id, " + fieldName),
"//*[@numFound='0']");
assertQ(req("q", fieldName + ":[" + Double.POSITIVE_INFINITY + " TO " + Double.POSITIVE_INFINITY + "}", "fl", "id, " + fieldName),
"//*[@numFound='0']");
} else {
assertQ(req("q", fieldName + ":[" + Float.MIN_VALUE + " TO " + Float.MIN_VALUE + "}", "fl", "id, " + fieldName),
"//*[@numFound='0']");
assertQ(req("q", fieldName + ":{" + Float.MAX_VALUE + " TO " + Float.MAX_VALUE + "]", "fl", "id, " + fieldName),
"//*[@numFound='0']");
assertQ(req("q", fieldName + ":{" + Float.NEGATIVE_INFINITY + " TO " + Float.NEGATIVE_INFINITY + "]", "fl", "id, " + fieldName),
"//*[@numFound='0']");
assertQ(req("q", fieldName + ":[" + Float.POSITIVE_INFINITY + " TO " + Float.POSITIVE_INFINITY + "}", "fl", "id, " + fieldName),
"//*[@numFound='0']");
}
doTestDoubleFloatRangeLimits(fieldName, testDouble);
}
private void doTestDoubleFloatRangeLimits(String fieldName, boolean testDouble) {
// POSITIVE/NEGATIVE_INFINITY toString is the same for Double and Float, it's OK to use this code for both cases
String positiveInfinity = String.valueOf(Double.POSITIVE_INFINITY);
String negativeInfinity = String.valueOf(Double.NEGATIVE_INFINITY);
String minVal = String.valueOf(testDouble?Double.MIN_VALUE:Float.MIN_VALUE);
String maxVal = String.valueOf(testDouble?Double.MAX_VALUE:Float.MAX_VALUE);
String negativeMinVal = "-" + minVal;
String negativeMaxVal = "-" + maxVal;
clearIndex();
assertU(adoc("id", "1", fieldName, minVal));
assertU(adoc("id", "2", fieldName, maxVal));
assertU(adoc("id", "3", fieldName, negativeInfinity));
assertU(adoc("id", "4", fieldName, positiveInfinity));
assertU(adoc("id", "5", fieldName, negativeMinVal));
assertU(adoc("id", "6", fieldName, negativeMaxVal));
assertU(commit());
//negative to negative
assertAllInclusiveExclusiveVariations(fieldName, "*", "-1", 2, 2, 2, 2);
assertAllInclusiveExclusiveVariations(fieldName, negativeInfinity, "-1", 1, 2, 1, 2);
assertAllInclusiveExclusiveVariations(fieldName, negativeMaxVal, negativeMinVal, 0, 1, 1, 2);
//negative to cero
assertAllInclusiveExclusiveVariations(fieldName, "*", "-0.0f", 3, 3, 3, 3);
assertAllInclusiveExclusiveVariations(fieldName, negativeInfinity, "-0.0f", 2, 3, 2, 3);
assertAllInclusiveExclusiveVariations(fieldName, negativeMinVal, "-0.0f", 0, 1, 0, 1);
assertAllInclusiveExclusiveVariations(fieldName, "*", "0", 3, 3, 3, 3);
assertAllInclusiveExclusiveVariations(fieldName, negativeInfinity, "0", 2, 3, 2, 3);
assertAllInclusiveExclusiveVariations(fieldName, negativeMinVal, "0", 0, 1, 0, 1);
//negative to positive
assertAllInclusiveExclusiveVariations(fieldName, "*", "1", 4, 4, 4, 4);
assertAllInclusiveExclusiveVariations(fieldName, "-1", "*", 4, 4, 4, 4);
assertAllInclusiveExclusiveVariations(fieldName, "-1", "1", 2, 2, 2, 2);
assertAllInclusiveExclusiveVariations(fieldName, "*", "*", 6, 6, 6, 6);
assertAllInclusiveExclusiveVariations(fieldName, "-1", positiveInfinity, 3, 3, 4, 4);
assertAllInclusiveExclusiveVariations(fieldName, negativeInfinity, "1", 3, 4, 3, 4);
assertAllInclusiveExclusiveVariations(fieldName, negativeInfinity, positiveInfinity, 4, 5, 5, 6);
assertAllInclusiveExclusiveVariations(fieldName, negativeMinVal, minVal, 0, 1, 1, 2);
assertAllInclusiveExclusiveVariations(fieldName, negativeMaxVal, maxVal, 2, 3, 3, 4);
//cero to positive
assertAllInclusiveExclusiveVariations(fieldName, "-0.0f", "*", 3, 3, 3, 3);
assertAllInclusiveExclusiveVariations(fieldName, "-0.0f", positiveInfinity, 2, 2, 3, 3);
assertAllInclusiveExclusiveVariations(fieldName, "-0.0f", minVal, 0, 0, 1, 1);
assertAllInclusiveExclusiveVariations(fieldName, "0", "*", 3, 3, 3, 3);
assertAllInclusiveExclusiveVariations(fieldName, "0", positiveInfinity, 2, 2, 3, 3);
assertAllInclusiveExclusiveVariations(fieldName, "0", minVal, 0, 0, 1, 1);
//positive to positive
assertAllInclusiveExclusiveVariations(fieldName, "1", "*", 2, 2, 2, 2);
assertAllInclusiveExclusiveVariations(fieldName, "1", positiveInfinity, 1, 1, 2, 2);
assertAllInclusiveExclusiveVariations(fieldName, minVal, maxVal, 0, 1, 1, 2);
// inverted limits
assertAllInclusiveExclusiveVariations(fieldName, "1", "-1", 0, 0, 0, 0);
assertAllInclusiveExclusiveVariations(fieldName, positiveInfinity, negativeInfinity, 0, 0, 0, 0);
assertAllInclusiveExclusiveVariations(fieldName, minVal, negativeMinVal, 0, 0, 0, 0);
// MatchNoDocs cases
assertAllInclusiveExclusiveVariations(fieldName, negativeInfinity, negativeInfinity, 0, 0, 0, 1);
assertAllInclusiveExclusiveVariations(fieldName, positiveInfinity, positiveInfinity, 0, 0, 0, 1);
clearIndex();
assertU(adoc("id", "1", fieldName, "0.0"));
assertU(adoc("id", "2", fieldName, "-0.0"));
assertU(commit());
assertAllInclusiveExclusiveVariations(fieldName, "*", "*", 2, 2, 2, 2);
assertAllInclusiveExclusiveVariations(fieldName, "*", "0", 1, 1, 2, 2);
assertAllInclusiveExclusiveVariations(fieldName, "0", "*", 0, 1, 0, 1);
assertAllInclusiveExclusiveVariations(fieldName, "*", "-0.0f", 0, 0, 1, 1);
assertAllInclusiveExclusiveVariations(fieldName, "-0.0f", "*", 1, 2, 1, 2);
assertAllInclusiveExclusiveVariations(fieldName, "-0.0f", "0", 0, 1, 1, 2);
}
private void assertAllInclusiveExclusiveVariations(String fieldName, String min, String max,
int countExclusiveExclusive,
int countInclusiveExclusive,
int countExclusiveInclusive,
int countInclusiveInclusive) {
assertQ(req("q", fieldName + ":{" + min + " TO " + max + "}", "fl", "id, " + fieldName),
"//*[@numFound='" + countExclusiveExclusive +"']");
assertQ(req("q", fieldName + ":[" + min + " TO " + max + "}", "fl", "id, " + fieldName),
"//*[@numFound='" + countInclusiveExclusive +"']");
assertQ(req("q", fieldName + ":{" + min + " TO " + max + "]", "fl", "id, " + fieldName),
"//*[@numFound='" + countExclusiveInclusive +"']");
assertQ(req("q", fieldName + ":[" + min + " TO " + max + "]", "fl", "id, " + fieldName),
"//*[@numFound='" + countInclusiveInclusive +"']");
}
private void doTestFloatPointFunctionQuery(String field) throws Exception {
assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof PointField);
int numVals = 10 * RANDOM_MULTIPLIER;

View File

@ -16,20 +16,33 @@
*/
package org.apache.solr.search;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.ResultContext;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.NumberType;
import org.apache.solr.schema.StrField;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.*;
public class TestRangeQuery extends SolrTestCaseJ4 {
private final static long DATE_START_TIME_RANDOM_TEST = 1499797224224L;
private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT);
@BeforeClass
public static void beforeClass() throws Exception {
@ -385,6 +398,292 @@ public class TestRangeQuery extends SolrTestCaseJ4 {
expectThrows(SyntaxError.class, () -> QParser.getParser("[A TO]", req("df", "text")).getQuery());
}
public void testCompareTypesRandomRangeQueries() throws Exception {
int cardinality = 10000;
Map<NumberType,String[]> types = new HashMap<>(); //single and multivalued field types
Map<NumberType,String[]> typesMv = new HashMap<>(); // multivalued field types only
types.put(NumberType.INTEGER, new String[]{"ti", "ti_dv", "ti_ni_dv", "i_p", "i_ni_p", "i_ndv_p", "tis", "tis_dv", "tis_ni_dv", "is_p", "is_ni_p", "is_ndv_p"});
types.put(NumberType.LONG, new String[]{"tl", "tl_dv", "tl_ni_dv", "l_p", "l_ni_p", "l_ndv_p", "tls", "tls_dv", "tls_ni_dv", "ls_p", "ls_ni_p", "ls_ndv_p"});
types.put(NumberType.FLOAT, new String[]{"tf", "tf_dv", "tf_ni_dv", "f_p", "f_ni_p", "f_ndv_p", "tfs", "tfs_dv", "tfs_ni_dv", "fs_p", "fs_ni_p", "fs_ndv_p"});
types.put(NumberType.DOUBLE, new String[]{"td", "td_dv", "td_ni_dv", "d_p", "d_ni_p", "d_ndv_p", "tds", "tds_dv", "tds_ni_dv", "ds_p", "ds_ni_p", "ds_ndv_p"});
types.put(NumberType.DATE, new String[]{"tdt", "tdt_dv", "tdt_ni_dv", "dt_p", "dt_ni_p", "dt_ndv_p", "tdts", "tdts_dv", "tdts_ni_dv", "dts_p", "dts_ni_p", "dts_ndv_p"});
typesMv.put(NumberType.INTEGER, new String[]{"tis", "tis_dv", "tis_ni_dv", "is_p", "is_ni_p", "is_ndv_p"});
typesMv.put(NumberType.LONG, new String[]{"tls", "tls_dv", "tls_ni_dv", "ls_p", "ls_ni_p", "ls_ndv_p"});
typesMv.put(NumberType.FLOAT, new String[]{"tfs", "tfs_dv", "tfs_ni_dv", "fs_p", "fs_ni_p", "fs_ndv_p"});
typesMv.put(NumberType.DOUBLE, new String[]{"tds", "tds_dv", "tds_ni_dv", "ds_p", "ds_ni_p", "ds_ndv_p"});
typesMv.put(NumberType.DATE, new String[]{"tdts", "tdts_dv", "tdts_ni_dv", "dts_p", "dts_ni_p", "dts_ndv_p"});
for (int i = 0; i < atLeast(500); i++) {
if (random().nextInt(50) == 0) {
//have some empty docs
assertU(adoc("id", String.valueOf(i)));
continue;
}
if (random().nextInt(100) == 0 && i > 0) {
//delete some docs
assertU(delI(String.valueOf(i - 1)));
}
SolrInputDocument document = new SolrInputDocument();
document.setField("id", i);
for (Map.Entry<NumberType,String[]> entry:types.entrySet()) {
NumberType type = entry.getKey();
String val = null;
List<String> vals = null;
switch (type) {
case DATE:
val = randomDate(cardinality);
vals = getRandomDates(random().nextInt(10), cardinality);
break;
case DOUBLE:
val = String.valueOf(randomDouble(cardinality));
vals = toStringList(getRandomDoubles(random().nextInt(10), cardinality));
break;
case FLOAT:
val = String.valueOf(randomFloat(cardinality));
vals = toStringList(getRandomFloats(random().nextInt(10), cardinality));
break;
case INTEGER:
val = String.valueOf(randomInt(cardinality));
vals = toStringList(getRandomInts(random().nextInt(10), cardinality));
break;
case LONG:
val = String.valueOf(randomLong(cardinality));
vals = toStringList(getRandomLongs(random().nextInt(10), cardinality));
break;
default:
throw new AssertionError();
}
// SingleValue
for (String fieldSuffix:entry.getValue()) {
document.setField("field_sv_" + fieldSuffix, val);
}
// MultiValue
for (String fieldSuffix:typesMv.get(type)) {
for (String value:vals) {
document.addField("field_mv_" + fieldSuffix, value);
}
}
}
assertU(adoc(document));
if (random().nextInt(50) == 0) {
assertU(commit());
}
}
assertU(commit());
String[][] possibleTypes = new String[types.size()][];
types.values().toArray(possibleTypes);
String[][] possibleTypesMv = new String[typesMv.size()][];
typesMv.values().toArray(possibleTypesMv);
for (int i = 0; i < atLeast(1000); i++) {
doTestQuery(cardinality, false, pickRandom(possibleTypes));
doTestQuery(cardinality, true, pickRandom(possibleTypesMv));
}
}
private void doTestQuery(int cardinality, boolean mv, String[] types) throws Exception {
String[] startOptions = new String[]{"{", "["};
String[] endOptions = new String[]{"}", "]"};
String[] qRange = getRandomRange(cardinality, types[0]);
String start = pickRandom(startOptions);
String end = pickRandom(endOptions);
long expectedHits = doRangeQuery(mv, start, end, types[0], qRange);
for (int i = 1; i < types.length; i++) {
assertEquals("Unexpected results from query when comparing " + types[0] + " with " + types[i] + " and query: " +
start + qRange[0] + " TO " + qRange[1] + end + "\n",
expectedHits, doRangeQuery(mv, start, end, types[i], qRange));
}
}
private long doRangeQuery(boolean mv, String start, String end, String field, String[] qRange) throws Exception {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("q", "field_" + (mv?"mv_":"sv_") + field + ":" + start + qRange[0] + " TO " + qRange[1] + end);
SolrQueryRequest req = req(params);
try {
return (long) h.queryAndResponse("", req).getToLog().get("hits");
} finally {
req.close();
}
}
private String[] getRandomRange(int max, String fieldName) {
Number[] values = new Number[2];
FieldType ft = h.getCore().getLatestSchema().getField("field_" + fieldName).getType();
if (ft.getNumberType() == null) {
assert ft instanceof StrField;
values[0] = randomInt(max);
values[1] = randomInt(max);
Arrays.sort(values, (o1, o2) -> String.valueOf(o1).compareTo(String.valueOf(o2)));
} else {
switch (ft.getNumberType()) {
case DOUBLE:
values[0] = randomDouble(max);
values[1] = randomDouble(max);
break;
case FLOAT:
values[0] = randomFloat(max);
values[1] = randomFloat(max);
break;
case INTEGER:
values[0] = randomInt(max);
values[1] = randomInt(max);
break;
case LONG:
values[0] = randomLong(max);
values[1] = randomLong(max);
break;
case DATE:
values[0] = randomMs(max);
values[1] = randomMs(max);
break;
default:
throw new AssertionError("Unexpected number type");
}
if (random().nextInt(100) >= 1) {// sometimes don't sort the values. Should result in 0 hits
Arrays.sort(values);
}
}
String[] stringValues = new String[2];
if (rarely()) {
stringValues[0] = "*";
} else {
if (ft.getNumberType() == NumberType.DATE) {
stringValues[0] = dateFormat.format(values[0]);
} else {
stringValues[0] = String.valueOf(values[0]);
}
}
if (rarely()) {
stringValues[1] = "*";
} else {
if (ft.getNumberType() == NumberType.DATE) {
stringValues[1] = dateFormat.format(values[1]);
} else {
stringValues[1] = String.valueOf(values[1]);
}
}
return stringValues;
}
// Helper methods
private String randomDate(int cardinality) {
return dateFormat.format(new Date(randomMs(cardinality)));
}
private List<String> getRandomDates(int numValues, int cardinality) {
List<String> vals = new ArrayList<>(numValues);
for (int i = 0; i < numValues;i++) {
vals.add(randomDate(cardinality));
}
return vals;
}
private List<Double> getRandomDoubles(int numValues, int cardinality) {
List<Double> vals = new ArrayList<>(numValues);
for (int i = 0; i < numValues;i++) {
vals.add(randomDouble(cardinality));
}
return vals;
}
private List<Float> getRandomFloats(int numValues, int cardinality) {
List<Float> vals = new ArrayList<>(numValues);
for (int i = 0; i < numValues;i++) {
vals.add(randomFloat(cardinality));
}
return vals;
}
private List<Integer> getRandomInts(int numValues, int cardinality) {
List<Integer> vals = new ArrayList<>(numValues);
for (int i = 0; i < numValues;i++) {
vals.add(randomInt(cardinality));
}
return vals;
}
private List<Long> getRandomLongs(int numValues, int cardinality) {
List<Long> vals = new ArrayList<>(numValues);
for (int i = 0; i < numValues;i++) {
vals.add(randomLong(cardinality));
}
return vals;
}
<T> List<String> toStringList(List<T> input) {
List<String> newList = new ArrayList<>(input.size());
for (T element:input) {
newList.add(String.valueOf(element));
}
return newList;
}
long randomMs(int cardinality) {
return DATE_START_TIME_RANDOM_TEST + random().nextInt(cardinality) * 1000 * (random().nextBoolean()?1:-1);
}
double randomDouble(int cardinality) {
if (rarely()) {
int num = random().nextInt(8);
if (num == 0) return Double.NEGATIVE_INFINITY;
if (num == 1) return Double.POSITIVE_INFINITY;
if (num == 2) return Double.MIN_VALUE;
if (num == 3) return Double.MAX_VALUE;
if (num == 4) return -Double.MIN_VALUE;
if (num == 5) return -Double.MAX_VALUE;
if (num == 6) return 0.0d;
if (num == 7) return -0.0d;
}
Double d = Double.NaN;
while (d.isNaN()) {
d = random().nextDouble();
}
return d * cardinality * (random().nextBoolean()?1:-1);
}
float randomFloat(int cardinality) {
if (rarely()) {
int num = random().nextInt(8);
if (num == 0) return Float.NEGATIVE_INFINITY;
if (num == 1) return Float.POSITIVE_INFINITY;
if (num == 2) return Float.MIN_VALUE;
if (num == 3) return Float.MAX_VALUE;
if (num == 4) return -Float.MIN_VALUE;
if (num == 5) return -Float.MAX_VALUE;
if (num == 6) return 0.0f;
if (num == 7) return -0.0f;
}
Float f = Float.NaN;
while (f.isNaN()) {
f = random().nextFloat();
}
return f * cardinality * (random().nextBoolean()?1:-1);
}
int randomInt(int cardinality) {
if (rarely()) {
int num = random().nextInt(2);
if (num == 0) return Integer.MAX_VALUE;
if (num == 1) return Integer.MIN_VALUE;
}
return random().nextInt(cardinality) * (random().nextBoolean()?1:-1);
}
long randomLong(int cardinality) {
if (rarely()) {
int num = random().nextInt(2);
if (num == 0) return Long.MAX_VALUE;
if (num == 1) return Long.MIN_VALUE;
}
return randomInt(cardinality);
}
static boolean sameDocs(String msg, DocSet a, DocSet b) {
DocIterator i = a.iterator();
// System.out.println("SIZES="+a.size() + "," + b.size());