Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add IntField, LongField, FloatField and DoubleField (#11997) 

This commit adds new IndexableFields that index both points and doc
values at once.

Closes #11199
This commit is contained in:
Francisco Fernández Castaño 2022-12-20 18:19:46 +01:00 committed by GitHub
parent 1412e559d9
commit 57201aa967
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 726 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lucene/CHANGES.txt
View File

@ -138,6 +138,10 @@ API Changes
* GITHUB#12004: Add new KnnByteVectorQuery for querying vector fields that are encoded as BYTE. Removes the ability to
use KnnVectorQuery against fields encoded as BYTE (Ben Trent)
* GITHUB#11997: Introduce IntField, LongField, FloatField and DoubleField.
These new fields index both 1D points and sorted numeric doc values and
provide best performance for filtering and sorting.
(Francisco Fernández Castaño, Adrien Grand)
New Features
---------------------
@ -149,6 +153,10 @@ New Features
* GITHUB#11999: MemoryIndex now supports stored fields. (Alan Woodward)
* GITHUB#11997: Add IntField, LongField, FloatField and DoubleField: easy to
use numeric fields that perform well both for filtering and sorting.
(Francisco Fernández Castaño)
Improvements
---------------------
* GITHUB#11778: Detailed part-of-speech information for particle(조사) and ending(어미) on Nori

141
lucene/core/src/java/org/apache/lucene/document/DoubleField.java Normal file
View File

@ -0,0 +1,141 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.document;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.PointRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedNumericSelector;
import org.apache.lucene.search.SortedNumericSortField;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
/**
* Field that stores a per-document <code>double</code> value for scoring, sorting or value
* retrieval and index the field for fast range filters. If you also need to store the value, you
* should add a separate {@link StoredField} instance. If you need more fine-grained control you can
* use {@link DoublePoint} and {@link DoubleDocValuesField}.
*
* <p>This field defines static factory methods for creating common queries:
*
* <ul>
* <li>{@link #newExactQuery(String, double)} for matching an exact 1D point.
* <li>{@link #newRangeQuery(String, double, double)} for matching a 1D range.
* </ul>
*
* @see PointValues
*/
public final class DoubleField extends Field {
private static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setDimensions(1, Double.BYTES);
FIELD_TYPE.setDocValuesType(DocValuesType.SORTED_NUMERIC);
FIELD_TYPE.freeze();
}
/**
* Creates a new DoubleField, indexing the provided point and storing it as a DocValue
*
* @param name field name
* @param value the double value
* @throws IllegalArgumentException if the field name or value is null.
*/
public DoubleField(String name, double value) {
super(name, FIELD_TYPE);
fieldsData = NumericUtils.doubleToSortableLong(value);
}
@Override
public BytesRef binaryValue() {
byte[] encodedPoint = new byte[Double.BYTES];
double value = getValueAsDouble();
DoublePoint.encodeDimension(value, encodedPoint, 0);
return new BytesRef(encodedPoint);
}
private double getValueAsDouble() {
return NumericUtils.sortableLongToDouble(numericValue().longValue());
}
@Override
public String toString() {
return getClass().getSimpleName() + " <" + name + ':' + getValueAsDouble() + '>';
}
@Override
public void setDoubleValue(double value) {
super.setLongValue(NumericUtils.doubleToSortableLong(value));
}
@Override
public void setLongValue(long value) {
throw new IllegalArgumentException("cannot change value type from Double to Long");
}
/**
* Create a query for matching an exact double value.
*
* @param field field name. must not be {@code null}.
* @param value exact value
* @throws IllegalArgumentException if {@code field} is null.
* @return a query matching documents with this exact value
*/
public static Query newExactQuery(String field, double value) {
return newRangeQuery(field, value, value);
}
/**
* Create a range query for double values.
*
* <p>You can have half-open ranges (which are in fact &lt;/&le; or &gt;/&ge; queries) by setting
* {@code lowerValue = Double.NEGATIVE_INFINITY} or {@code upperValue = Double.POSITIVE_INFINITY}.
*
* <p>Range comparisons are consistent with {@link Double#compareTo(Double)}.
*
* @param field field name. must not be {@code null}.
* @param lowerValue lower portion of the range (inclusive).
* @param upperValue upper portion of the range (inclusive).
* @throws IllegalArgumentException if {@code field} is null.
* @return a query matching documents within this range.
*/
public static Query newRangeQuery(String field, double lowerValue, double upperValue) {
PointRangeQuery.checkArgs(field, lowerValue, upperValue);
return new IndexOrDocValuesQuery(
DoublePoint.newRangeQuery(field, lowerValue, upperValue),
SortedNumericDocValuesField.newSlowRangeQuery(
field,
NumericUtils.doubleToSortableLong(lowerValue),
NumericUtils.doubleToSortableLong(upperValue)));
}
/**
* Create a new {@link SortField} for double values.
*
* @param field field name. must not be {@code null}.
* @param reverse true if natural order should be reversed.
* @param selector custom selector type for choosing the sort value from the set.
*/
public static SortField newSortField(
String field, boolean reverse, SortedNumericSelector.Type selector) {
return new SortedNumericSortField(field, SortField.Type.DOUBLE, reverse, selector);
}
}

141
lucene/core/src/java/org/apache/lucene/document/FloatField.java Normal file
View File

@ -0,0 +1,141 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.document;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.PointRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedNumericSelector;
import org.apache.lucene.search.SortedNumericSortField;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
/**
* Field that stores a per-document <code>float</code> value for scoring, sorting or value retrieval
* and index the field for fast range filters. If you also need to store the value, you should add a
* separate {@link StoredField} instance. If you need more fine-grained control you can use {@link
* FloatPoint} and {@link FloatDocValuesField}.
*
* <p>This field defines static factory methods for creating common queries:
*
* <ul>
* <li>{@link #newExactQuery(String, float)} for matching an exact 1D point.
* <li>{@link #newRangeQuery(String, float, float)} for matching a 1D range.
* </ul>
*
* @see PointValues
*/
public final class FloatField extends Field {
private static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setDimensions(1, Float.BYTES);
FIELD_TYPE.setDocValuesType(DocValuesType.SORTED_NUMERIC);
FIELD_TYPE.freeze();
}
/**
* Creates a new FloatField, indexing the provided point and storing it as a DocValue
*
* @param name field name
* @param value the float value
* @throws IllegalArgumentException if the field name or value is null.
*/
public FloatField(String name, float value) {
super(name, FIELD_TYPE);
fieldsData = (long) NumericUtils.floatToSortableInt(value);
}
@Override
public BytesRef binaryValue() {
byte[] encodedPoint = new byte[Float.BYTES];
float value = getValueAsFloat();
FloatPoint.encodeDimension(value, encodedPoint, 0);
return new BytesRef(encodedPoint);
}
private float getValueAsFloat() {
return NumericUtils.sortableIntToFloat(numericValue().intValue());
}
@Override
public String toString() {
return getClass().getSimpleName() + " <" + name + ':' + getValueAsFloat() + '>';
}
@Override
public void setFloatValue(float value) {
super.setLongValue(NumericUtils.floatToSortableInt(value));
}
@Override
public void setLongValue(long value) {
throw new IllegalArgumentException("cannot change value type from Float to Long");
}
/**
* Create a query for matching an exact float value.
*
* @param field field name. must not be {@code null}.
* @param value exact value
* @throws IllegalArgumentException if {@code field} is null.
* @return a query matching documents with this exact value
*/
public static Query newExactQuery(String field, float value) {
return newRangeQuery(field, value, value);
}
/**
* Create a range query for float values.
*
* <p>You can have half-open ranges (which are in fact &lt;/&le; or &gt;/&ge; queries) by setting
* {@code lowerValue = Float.NEGATIVE_INFINITY} or {@code upperValue = Float.POSITIVE_INFINITY}.
*
* <p>Range comparisons are consistent with {@link Float#compareTo(Float)}.
*
* @param field field name. must not be {@code null}.
* @param lowerValue lower portion of the range (inclusive).
* @param upperValue upper portion of the range (inclusive).
* @throws IllegalArgumentException if {@code field} is null.
* @return a query matching documents within this range.
*/
public static Query newRangeQuery(String field, float lowerValue, float upperValue) {
PointRangeQuery.checkArgs(field, lowerValue, upperValue);
return new IndexOrDocValuesQuery(
FloatPoint.newRangeQuery(field, lowerValue, upperValue),
SortedNumericDocValuesField.newSlowRangeQuery(
field,
NumericUtils.floatToSortableInt(lowerValue),
NumericUtils.floatToSortableInt(upperValue)));
}
/**
* Create a new {@link SortField} for float values.
*
* @param field field name. must not be {@code null}.
* @param reverse true if natural order should be reversed.
* @param selector custom selector type for choosing the sort value from the set.
*/
public static SortField newSortField(
String field, boolean reverse, SortedNumericSelector.Type selector) {
return new SortedNumericSortField(field, SortField.Type.FLOAT, reverse, selector);
}
}

124
lucene/core/src/java/org/apache/lucene/document/IntField.java Normal file
View File

@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.document;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.PointRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedNumericSelector;
import org.apache.lucene.search.SortedNumericSortField;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
/**
* Field that stores a per-document <code>int</code> value for scoring, sorting or value retrieval
* and index the field for fast range filters. If you also need to store the value, you should add a
* separate {@link StoredField} instance. If you need more fine-grained control you can use {@link
* IntPoint} and {@link NumericDocValuesField} or {@link SortedNumericDocValuesField}.
*
* <p>This field defines static factory methods for creating common queries:
*
* <ul>
* <li>{@link #newExactQuery(String, int)} for matching an exact 1D point.
* <li>{@link #newRangeQuery(String, int, int)} for matching a 1D range.
* </ul>
*
* @see PointValues
*/
public final class IntField extends Field {
private static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setDimensions(1, Integer.BYTES);
FIELD_TYPE.setDocValuesType(DocValuesType.SORTED_NUMERIC);
FIELD_TYPE.freeze();
}
/**
* Creates a new IntField, indexing the provided point and storing it as a DocValue
*
* @param name field name
* @param value the int value
* @throws IllegalArgumentException if the field name or value is null.
*/
public IntField(String name, int value) {
super(name, FIELD_TYPE);
fieldsData = value;
}
@Override
public BytesRef binaryValue() {
var bytes = new byte[Integer.BYTES];
NumericUtils.intToSortableBytes((Integer) fieldsData, bytes, 0);
return new BytesRef(bytes);
}
@Override
public String toString() {
return getClass().getSimpleName() + " <" + name + ':' + fieldsData + '>';
}
/**
* Create a query for matching an exact integer value.
*
* @param field field name. must not be {@code null}.
* @param value exact value
* @throws IllegalArgumentException if {@code field} is null.
* @return a query matching documents with this exact value
*/
public static Query newExactQuery(String field, int value) {
return newRangeQuery(field, value, value);
}
/**
* Create a range query for integer values.
*
* <p>You can have half-open ranges (which are in fact &lt;/&le; or &gt;/&ge; queries) by setting
* {@code lowerValue = Integer.MIN_VALUE} or {@code upperValue = Integer.MAX_VALUE}.
*
* <p>Ranges are inclusive. For exclusive ranges, pass {@code Math.addExact(lowerValue, 1)} or
* {@code Math.addExact(upperValue, -1)}.
*
* @param field field name. must not be {@code null}.
* @param lowerValue lower portion of the range (inclusive).
* @param upperValue upper portion of the range (inclusive).
* @throws IllegalArgumentException if {@code field} is null.
* @return a query matching documents within this range.
*/
public static Query newRangeQuery(String field, int lowerValue, int upperValue) {
PointRangeQuery.checkArgs(field, lowerValue, upperValue);
return new IndexOrDocValuesQuery(
IntPoint.newRangeQuery(field, lowerValue, upperValue),
SortedNumericDocValuesField.newSlowRangeQuery(field, lowerValue, upperValue));
}
/**
* Create a new {@link SortField} for int values.
*
* @param field field name. must not be {@code null}.
* @param reverse true if natural order should be reversed.
* @param selector custom selector type for choosing the sort value from the set.
*/
public static SortField newSortField(
String field, boolean reverse, SortedNumericSelector.Type selector) {
return new SortedNumericSortField(field, SortField.Type.INT, reverse, selector);
}
}

145
lucene/core/src/java/org/apache/lucene/document/LongField.java Normal file
View File

@ -0,0 +1,145 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.document;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.PointRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedNumericSelector;
import org.apache.lucene.search.SortedNumericSortField;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
/**
* Field that stores a per-document <code>long</code> value for scoring, sorting or value retrieval
* and index the field for fast range filters. If you also need to store the value, you should add a
* separate {@link StoredField} instance. If you need more fine-grained control you can use {@link
* LongPoint} and {@link NumericDocValuesField} or {@link SortedNumericDocValuesField}.
*
* <p>This field defines static factory methods for creating common queries:
*
* <ul>
* <li>{@link #newExactQuery(String, long)} for matching an exact 1D point.
* <li>{@link #newRangeQuery(String, long, long)} for matching a 1D range.
* </ul>
*
* @see PointValues
*/
public final class LongField extends Field {
private static final FieldType FIELD_TYPE = new FieldType();
static {
FIELD_TYPE.setDimensions(1, Long.BYTES);
FIELD_TYPE.setDocValuesType(DocValuesType.SORTED_NUMERIC);
FIELD_TYPE.freeze();
}
/**
* Creates a new LongField, indexing the provided point and storing it as a DocValue
*
* @param name field name
* @param value the long value
* @throws IllegalArgumentException if the field name or value is null.
*/
public LongField(String name, long value) {
super(name, FIELD_TYPE);
fieldsData = value;
}
@Override
public BytesRef binaryValue() {
var bytes = new byte[Long.BYTES];
NumericUtils.longToSortableBytes((Long) fieldsData, bytes, 0);
return new BytesRef(bytes);
}
@Override
public String toString() {
return getClass().getSimpleName() + " <" + name + ':' + fieldsData + '>';
}
/**
* Create a query for matching an exact long value.
*
* @param field field name. must not be {@code null}.
* @param value exact value
* @throws IllegalArgumentException if {@code field} is null.
* @return a query matching documents with this exact value
*/
public static Query newExactQuery(String field, long value) {
return newRangeQuery(field, value, value);
}
/**
* Create a range query for long values.
*
* <p>You can have half-open ranges (which are in fact &lt;/&le; or &gt;/&ge; queries) by setting
* {@code lowerValue = Long.MIN_VALUE} or {@code upperValue = Long.MAX_VALUE}.
*
* <p>Ranges are inclusive. For exclusive ranges, pass {@code Math.addExact(lowerValue, 1)} or
* {@code Math.addExact(upperValue, -1)}.
*
* @param field field name. must not be {@code null}.
* @param lowerValue lower portion of the range (inclusive).
* @param upperValue upper portion of the range (inclusive).
* @throws IllegalArgumentException if {@code field} is null.
* @return a query matching documents within this range.
*/
public static Query newRangeQuery(String field, long lowerValue, long upperValue) {
PointRangeQuery.checkArgs(field, lowerValue, upperValue);
return new IndexOrDocValuesQuery(
LongPoint.newRangeQuery(field, lowerValue, upperValue),
SortedNumericDocValuesField.newSlowRangeQuery(field, lowerValue, upperValue));
}
/**
* Create a new {@link SortField} for long values.
*
* @param field field name. must not be {@code null}.
* @param reverse true if natural order should be reversed.
* @param selector custom selector type for choosing the sort value from the set.
*/
public static SortField newSortField(
String field, boolean reverse, SortedNumericSelector.Type selector) {
return new SortedNumericSortField(field, SortField.Type.LONG, reverse, selector);
}
/**
* Returns a query that scores documents based on their distance to {@code origin}: {@code score =
* weight * pivotDistance / (pivotDistance + distance)}, ie. score is in the {@code [0, weight]}
* range, is equal to {@code weight} when the document's value is equal to {@code origin} and is
* equal to {@code weight/2} when the document's value is distant of {@code pivotDistance} from
* {@code origin}. In case of multi-valued fields, only the closest point to {@code origin} will
* be considered. This query is typically useful to boost results based on recency by adding this
* query to a {@link Occur#SHOULD} clause of a {@link BooleanQuery}.
*/
public static Query newDistanceFeatureQuery(
String field, float weight, long origin, long pivotDistance) {
Query query = new LongDistanceFeatureQuery(field, origin, pivotDistance);
if (weight != 1f) {
query = new BoostQuery(query, weight);
}
return query;
}
}

11
lucene/core/src/java/org/apache/lucene/document/LongPoint.java
View File

@ -21,7 +21,6 @@ import java.util.Collection;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.PointInSetQuery;
import org.apache.lucene.search.PointRangeQuery;
import org.apache.lucene.search.Query;
@ -312,14 +311,12 @@ public final class LongPoint extends Field {
* {@code weight/2} when the document's value is distant of {@code pivotDistance} from {@code
* origin}. In case of multi-valued fields, only the closest point to {@code origin} will be
* considered. This query is typically useful to boost results based on recency by adding this
* query to a {@link Occur#SHOULD} clause of a {@link BooleanQuery}.
* query to a {@link Occur#SHOULD} clause of a {@link BooleanQuery}. @Deprecated Use {@link
* LongField#newDistanceFeatureQuery}
*/
@Deprecated
public static Query newDistanceFeatureQuery(
String field, float weight, long origin, long pivotDistance) {
Query query = new LongDistanceFeatureQuery(field, origin, pivotDistance);
if (weight != 1f) {
query = new BoostQuery(query, weight);
}
return query;
return LongField.newDistanceFeatureQuery(field, weight, origin, pivotDistance);
}
}

90
lucene/core/src/test/org/apache/lucene/document/TestField.java
View File

@ -36,6 +36,7 @@ import org.apache.lucene.tests.analysis.Token;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
// sanity check some basics of fields
public class TestField extends LuceneTestCase {
@ -211,6 +212,95 @@ public class TestField extends LuceneTestCase {
assertEquals("IntPoint <foo:6,7>", field.toString());
}
public void testIntField() throws Exception {
IntField field = new IntField("foo", 12);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
trySetDoubleValue(field);
field.setIntValue(6);
trySetLongValue(field);
trySetFloatValue(field);
trySetLongValue(field);
trySetReaderValue(field);
trySetShortValue(field);
trySetStringValue(field);
trySetTokenStreamValue(field);
assertEquals(6, field.numericValue().intValue());
assertEquals(6, NumericUtils.sortableBytesToInt(field.binaryValue().bytes, 0));
assertEquals("IntField <foo:6>", field.toString());
}
public void testLongField() throws Exception {
LongField field = new LongField("foo", 12);
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
trySetDoubleValue(field);
trySetIntValue(field);
field.setLongValue(6);
trySetFloatValue(field);
trySetReaderValue(field);
trySetShortValue(field);
trySetStringValue(field);
trySetTokenStreamValue(field);
assertEquals(6L, field.numericValue().longValue());
assertEquals(6L, NumericUtils.sortableBytesToLong(field.binaryValue().bytes, 0));
assertEquals("LongField <foo:6>", field.toString());
}
public void testFloatField() throws Exception {
FloatField field = new FloatField("foo", 12.6f);
assertEquals(12.6f, NumericUtils.sortableIntToFloat(field.numericValue().intValue()), 0.0f);
assertEquals(12.6f, FloatPoint.decodeDimension(field.binaryValue().bytes, 0), 0.0f);
assertEquals("FloatField <foo:12.6>", field.toString());
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
trySetDoubleValue(field);
trySetIntValue(field);
trySetLongValue(field);
field.setFloatValue(-28.8f);
trySetReaderValue(field);
trySetShortValue(field);
trySetStringValue(field);
trySetTokenStreamValue(field);
assertEquals(-28.8f, NumericUtils.sortableIntToFloat(field.numericValue().intValue()), 0.0f);
assertEquals(-28.8f, FloatPoint.decodeDimension(field.binaryValue().bytes, 0), 0.0f);
assertEquals("FloatField <foo:-28.8>", field.toString());
}
public void testDoubleField() throws Exception {
DoubleField field = new DoubleField("foo", 12.7);
assertEquals(12.7, NumericUtils.sortableLongToDouble(field.numericValue().longValue()), 0.0f);
assertEquals(12.7, DoublePoint.decodeDimension(field.binaryValue().bytes, 0), 0.0f);
assertEquals("DoubleField <foo:12.7>", field.toString());
trySetByteValue(field);
trySetBytesValue(field);
trySetBytesRefValue(field);
trySetIntValue(field);
trySetLongValue(field);
trySetFloatValue(field);
field.setDoubleValue(-28.8);
trySetReaderValue(field);
trySetShortValue(field);
trySetStringValue(field);
trySetTokenStreamValue(field);
assertEquals(-28.8, NumericUtils.sortableLongToDouble(field.numericValue().longValue()), 0.0f);
assertEquals(-28.8, DoublePoint.decodeDimension(field.binaryValue().bytes, 0), 0.0f);
assertEquals("DoubleField <foo:-28.8>", field.toString());
}
public void testNumericDocValuesField() throws Exception {
NumericDocValuesField field = new NumericDocValuesField("foo", 5L);

108
lucene/core/src/test/org/apache/lucene/document/TestLongDistanceFeatureQuery.java
View File

@ -36,20 +36,20 @@ import org.apache.lucene.tests.util.LuceneTestCase;
public class TestLongDistanceFeatureQuery extends LuceneTestCase {
public void testEqualsAndHashcode() {
Query q1 = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5);
Query q2 = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5);
Query q1 = LongField.newDistanceFeatureQuery("foo", 3, 10, 5);
Query q2 = LongField.newDistanceFeatureQuery("foo", 3, 10, 5);
QueryUtils.checkEqual(q1, q2);
Query q3 = LongPoint.newDistanceFeatureQuery("bar", 3, 10, 5);
Query q3 = LongField.newDistanceFeatureQuery("bar", 3, 10, 5);
QueryUtils.checkUnequal(q1, q3);
Query q4 = LongPoint.newDistanceFeatureQuery("foo", 4, 10, 5);
Query q4 = LongField.newDistanceFeatureQuery("foo", 4, 10, 5);
QueryUtils.checkUnequal(q1, q4);
Query q5 = LongPoint.newDistanceFeatureQuery("foo", 3, 9, 5);
Query q5 = LongField.newDistanceFeatureQuery("foo", 3, 9, 5);
QueryUtils.checkUnequal(q1, q5);
Query q6 = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 6);
Query q6 = LongField.newDistanceFeatureQuery("foo", 3, 10, 6);
QueryUtils.checkUnequal(q1, q6);
}
@ -61,35 +61,28 @@ public class TestLongDistanceFeatureQuery extends LuceneTestCase {
dir,
newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean())));
Document doc = new Document();
LongPoint point = new LongPoint("foo", 0L);
doc.add(point);
NumericDocValuesField docValue = new NumericDocValuesField("foo", 0L);
doc.add(docValue);
LongField field = new LongField("foo", 0L);
doc.add(field);
point.setLongValue(3);
docValue.setLongValue(3);
field.setLongValue(3);
w.addDocument(doc);
point.setLongValue(12);
docValue.setLongValue(12);
field.setLongValue(12);
w.addDocument(doc);
point.setLongValue(8);
docValue.setLongValue(8);
field.setLongValue(8);
w.addDocument(doc);
point.setLongValue(-1);
docValue.setLongValue(-1);
field.setLongValue(-1);
w.addDocument(doc);
point.setLongValue(7);
docValue.setLongValue(7);
field.setLongValue(7);
w.addDocument(doc);
DirectoryReader reader = w.getReader();
IndexSearcher searcher = newSearcher(reader);
Query q = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5);
Query q = LongField.newDistanceFeatureQuery("foo", 3, 10, 5);
CollectorManager<TopScoreDocCollector, TopDocs> manager =
TopScoreDocCollector.createSharedManager(2, null, 1);
TopDocs topHits = searcher.search(q, manager);
@ -103,7 +96,7 @@ public class TestLongDistanceFeatureQuery extends LuceneTestCase {
},
topHits.scoreDocs);
q = LongPoint.newDistanceFeatureQuery("foo", 3, 7, 5);
q = LongField.newDistanceFeatureQuery("foo", 3, 7, 5);
manager = TopScoreDocCollector.createSharedManager(2, null, 1);
topHits = searcher.search(q, manager);
assertEquals(2, topHits.scoreDocs.length);
@ -130,35 +123,28 @@ public class TestLongDistanceFeatureQuery extends LuceneTestCase {
dir,
newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean())));
Document doc = new Document();
LongPoint point = new LongPoint("foo", 0L);
doc.add(point);
NumericDocValuesField docValue = new NumericDocValuesField("foo", 0L);
doc.add(docValue);
LongField field = new LongField("foo", 0L);
doc.add(field);
point.setLongValue(3);
docValue.setLongValue(3);
field.setLongValue(3);
w.addDocument(doc);
point.setLongValue(12);
docValue.setLongValue(12);
field.setLongValue(12);
w.addDocument(doc);
point.setLongValue(-10);
docValue.setLongValue(-10);
field.setLongValue(-10);
w.addDocument(doc);
point.setLongValue(Long.MAX_VALUE);
docValue.setLongValue(Long.MAX_VALUE);
field.setLongValue(Long.MAX_VALUE);
w.addDocument(doc);
point.setLongValue(Long.MIN_VALUE);
docValue.setLongValue(Long.MIN_VALUE);
field.setLongValue(Long.MIN_VALUE);
w.addDocument(doc);
DirectoryReader reader = w.getReader();
IndexSearcher searcher = newSearcher(reader);
Query q = LongPoint.newDistanceFeatureQuery("foo", 3, Long.MAX_VALUE - 1, 100);
Query q = LongField.newDistanceFeatureQuery("foo", 3, Long.MAX_VALUE - 1, 100);
CollectorManager<TopScoreDocCollector, TopDocs> manager =
TopScoreDocCollector.createSharedManager(2, null, 1);
TopDocs topHits = searcher.search(q, manager);
@ -180,7 +166,7 @@ public class TestLongDistanceFeatureQuery extends LuceneTestCase {
},
topHits.scoreDocs);
q = LongPoint.newDistanceFeatureQuery("foo", 3, Long.MIN_VALUE + 1, 100);
q = LongField.newDistanceFeatureQuery("foo", 3, Long.MIN_VALUE + 1, 100);
topHits = searcher.search(q, manager);
assertEquals(2, topHits.scoreDocs.length);
CheckHits.checkExplanations(q, "", searcher);
@ -210,7 +196,7 @@ public class TestLongDistanceFeatureQuery extends LuceneTestCase {
IndexReader reader = new MultiReader();
IndexSearcher searcher = newSearcher(reader);
Query q = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5);
Query q = LongField.newDistanceFeatureQuery("foo", 3, 10, 5);
TopDocs topHits = searcher.search(q, 2);
assertEquals(0, topHits.totalHits.value);
}
@ -223,25 +209,21 @@ public class TestLongDistanceFeatureQuery extends LuceneTestCase {
dir,
newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean())));
Document doc = new Document();
LongPoint point = new LongPoint("foo", 0L);
doc.add(point);
NumericDocValuesField docValue = new NumericDocValuesField("foo", 0L);
doc.add(docValue);
LongField field = new LongField("foo", 0L);
doc.add(field);
point.setLongValue(3);
docValue.setLongValue(3);
field.setLongValue(3);
w.addDocument(doc);
w.addDocument(new Document());
point.setLongValue(7);
docValue.setLongValue(7);
field.setLongValue(7);
w.addDocument(doc);
DirectoryReader reader = w.getReader();
IndexSearcher searcher = newSearcher(reader);
Query q = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5);
Query q = LongField.newDistanceFeatureQuery("foo", 3, 10, 5);
CollectorManager<TopScoreDocCollector, TopDocs> manager =
TopScoreDocCollector.createSharedManager(3, null, 1);
TopDocs topHits = searcher.search(q, manager);
@ -272,43 +254,38 @@ public class TestLongDistanceFeatureQuery extends LuceneTestCase {
Document doc = new Document();
for (long v : new long[] {3, 1000, Long.MAX_VALUE}) {
doc.add(new LongPoint("foo", v));
doc.add(new SortedNumericDocValuesField("foo", v));
doc.add(new LongField("foo", v));
}
w.addDocument(doc);
doc = new Document();
for (long v : new long[] {-100, 12, 999}) {
doc.add(new LongPoint("foo", v));
doc.add(new SortedNumericDocValuesField("foo", v));
doc.add(new LongField("foo", v));
}
w.addDocument(doc);
doc = new Document();
for (long v : new long[] {Long.MIN_VALUE, -1000, 8}) {
doc.add(new LongPoint("foo", v));
doc.add(new SortedNumericDocValuesField("foo", v));
doc.add(new LongField("foo", v));
}
w.addDocument(doc);
doc = new Document();
for (long v : new long[] {-1}) {
doc.add(new LongPoint("foo", v));
doc.add(new SortedNumericDocValuesField("foo", v));
doc.add(new LongField("foo", v));
}
w.addDocument(doc);
doc = new Document();
for (long v : new long[] {Long.MIN_VALUE, 7}) {
doc.add(new LongPoint("foo", v));
doc.add(new SortedNumericDocValuesField("foo", v));
doc.add(new LongField("foo", v));
}
w.addDocument(doc);
DirectoryReader reader = w.getReader();
IndexSearcher searcher = newSearcher(reader);
Query q = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5);
Query q = LongField.newDistanceFeatureQuery("foo", 3, 10, 5);
CollectorManager<TopScoreDocCollector, TopDocs> manager =
TopScoreDocCollector.createSharedManager(2, null, 1);
TopDocs topHits = searcher.search(q, manager);
@ -322,7 +299,7 @@ public class TestLongDistanceFeatureQuery extends LuceneTestCase {
},
topHits.scoreDocs);
q = LongPoint.newDistanceFeatureQuery("foo", 3, 7, 5);
q = LongField.newDistanceFeatureQuery("foo", 3, 7, 5);
manager = TopScoreDocCollector.createSharedManager(2, null, 1);
topHits = searcher.search(q, manager);
assertEquals(2, topHits.scoreDocs.length);
@ -347,16 +324,13 @@ public class TestLongDistanceFeatureQuery extends LuceneTestCase {
new IndexWriter(
dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean())));
Document doc = new Document();
LongPoint point = new LongPoint("foo", 0L);
doc.add(point);
NumericDocValuesField docValue = new NumericDocValuesField("foo", 0L);
doc.add(docValue);
LongField field = new LongField("foo", 0L);
doc.add(field);
int numDocs = atLeast(10000);
for (int i = 0; i < numDocs; ++i) {
long v = random().nextLong();
point.setLongValue(v);
docValue.setLongValue(v);
field.setLongValue(v);
w.addDocument(doc);
}
@ -370,7 +344,7 @@ public class TestLongDistanceFeatureQuery extends LuceneTestCase {
pivotDistance = random().nextLong();
} while (pivotDistance <= 0);
float boost = (1 + random().nextInt(10)) / 3f;
Query q = LongPoint.newDistanceFeatureQuery("foo", boost, origin, pivotDistance);
Query q = LongField.newDistanceFeatureQuery("foo", boost, origin, pivotDistance);
CheckHits.checkTopScores(random(), q, searcher);
}

10
lucene/core/src/test/org/apache/lucene/search/TestIndexOrDocValuesQuery.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
@ -113,18 +114,15 @@ public class TestIndexOrDocValuesQuery extends LuceneTestCase {
if (i < 1000) {
doc.add(new StringField("f1", "bar", Store.NO));
for (int j = 0; j < 500; j++) {
doc.add(new LongPoint("f2", 42L));
doc.add(new SortedNumericDocValuesField("f2", 42L));
doc.add(new LongField("f2", 42L));
}
} else if (i == 1001) {
doc.add(new StringField("f1", "foo", Store.NO));
doc.add(new LongPoint("f2", 2L));
doc.add(new SortedNumericDocValuesField("f2", 42L));
doc.add(new LongField("f2", 2L));
} else {
doc.add(new StringField("f1", "bar", Store.NO));
for (int j = 0; j < 100; j++) {
doc.add(new LongPoint("f2", 2L));
doc.add(new SortedNumericDocValuesField("f2", 2L));
doc.add(new LongField("f2", 2L));
}
}
w.addDocument(doc);

14
lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java
View File

@ -31,10 +31,10 @@ import org.apache.lucene.document.FloatDocValuesField;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.IntRange;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
@ -809,10 +809,8 @@ public class TestSortOptimization extends LuceneTestCase {
int value = random().nextInt();
int value2 = random().nextInt();
final Document doc = new Document();
doc.add(new SortedNumericDocValuesField("my_field", value));
doc.add(new SortedNumericDocValuesField("my_field", value2));
doc.add(new LongPoint("my_field", value));
doc.add(new LongPoint("my_field", value2));
doc.add(new LongField("my_field", value));
doc.add(new LongField("my_field", value2));
writer.addDocument(doc);
}
final IndexReader reader = DirectoryReader.open(writer);
@ -823,12 +821,10 @@ public class TestSortOptimization extends LuceneTestCase {
SortedNumericSelector.Type type =
RandomPicks.randomFrom(random(), SortedNumericSelector.Type.values());
boolean reverse = random().nextBoolean();
final SortField sortField =
new SortedNumericSortField("my_field", SortField.Type.LONG, reverse, type);
final SortField sortField = LongField.newSortField("my_field", reverse, type);
sortField.setOptimizeSortWithIndexedData(false);
final Sort sort = new Sort(sortField); // sort without sort optimization
final SortField sortField2 =
new SortedNumericSortField("my_field", SortField.Type.LONG, reverse, type);
final SortField sortField2 = LongField.newSortField("my_field", reverse, type);
final Sort sort2 = new Sort(sortField2); // sort with sort optimization
Query query = new MatchAllDocsQuery();
final int totalHitsThreshold = 3;

45
lucene/core/src/test/org/apache/lucene/search/TestSortedNumericSortField.java
View File

@ -17,15 +17,16 @@
package org.apache.lucene.search;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.NumericUtils;
/** Simple tests for SortedNumericSortField */
public class TestSortedNumericSortField extends LuceneTestCase {
@ -79,12 +80,12 @@ public class TestSortedNumericSortField extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new SortedNumericDocValuesField("value", 5));
doc.add(new IntField("value", 5));
doc.add(newStringField("id", "2", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new SortedNumericDocValuesField("value", 3));
doc.add(new SortedNumericDocValuesField("value", 7));
doc.add(new IntField("value", 3));
doc.add(new IntField("value", 7));
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
@ -107,12 +108,12 @@ public class TestSortedNumericSortField extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new SortedNumericDocValuesField("value", 3));
doc.add(new SortedNumericDocValuesField("value", 7));
doc.add(new IntField("value", 3));
doc.add(new IntField("value", 7));
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new SortedNumericDocValuesField("value", 5));
doc.add(new IntField("value", 5));
doc.add(newStringField("id", "2", Field.Store.YES));
writer.addDocument(doc);
@ -136,12 +137,12 @@ public class TestSortedNumericSortField extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new SortedNumericDocValuesField("value", 5));
doc.add(new IntField("value", 5));
doc.add(newStringField("id", "2", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new SortedNumericDocValuesField("value", 3));
doc.add(new SortedNumericDocValuesField("value", 7));
doc.add(new IntField("value", 3));
doc.add(new IntField("value", 7));
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
@ -171,12 +172,12 @@ public class TestSortedNumericSortField extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new SortedNumericDocValuesField("value", 5));
doc.add(new IntField("value", 5));
doc.add(newStringField("id", "2", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new SortedNumericDocValuesField("value", 3));
doc.add(new SortedNumericDocValuesField("value", 7));
doc.add(new IntField("value", 3));
doc.add(new IntField("value", 7));
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
@ -206,11 +207,11 @@ public class TestSortedNumericSortField extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new SortedNumericDocValuesField("value", 5));
doc.add(new IntField("value", 5));
doc.add(newStringField("id", "2", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new SortedNumericDocValuesField("value", 3));
doc.add(new IntField("value", 3));
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
@ -233,12 +234,12 @@ public class TestSortedNumericSortField extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new SortedNumericDocValuesField("value", NumericUtils.floatToSortableInt(-3f)));
doc.add(new FloatField("value", -3f));
doc.add(newStringField("id", "2", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new SortedNumericDocValuesField("value", NumericUtils.floatToSortableInt(-5f)));
doc.add(new SortedNumericDocValuesField("value", NumericUtils.floatToSortableInt(7f)));
doc.add(new FloatField("value", -5f));
doc.add(new FloatField("value", 7f));
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
@ -261,12 +262,12 @@ public class TestSortedNumericSortField extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new SortedNumericDocValuesField("value", NumericUtils.doubleToSortableLong(-3d)));
doc.add(new DoubleField("value", -3d));
doc.add(newStringField("id", "2", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new SortedNumericDocValuesField("value", NumericUtils.doubleToSortableLong(-5d)));
doc.add(new SortedNumericDocValuesField("value", NumericUtils.doubleToSortableLong(7d)));
doc.add(new DoubleField("value", -5d));
doc.add(new DoubleField("value", 7d));
doc.add(newStringField("id", "1", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();