From a969dad43e9027552ddccec09360dd022051c253 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 14 Feb 2017 15:57:12 +0100 Subject: [PATCH] Integrate IndexOrDocValuesQuery. (#23119) This gives Lucene the choice to use index/point-based queries or doc-values-based queries depending on which one is more efficient. This commit integrates this feature for: - long/integer/short/byte/double/float/half_float/scaled_float ranges, - date ranges, - geo bounding box queries, - geo distance queries. --- .../index/mapper/DateFieldMapper.java | 8 +- .../index/mapper/NumberFieldMapper.java | 72 ++++++-- .../index/mapper/ScaledFloatFieldMapper.java | 2 +- .../query/GeoBoundingBoxQueryBuilder.java | 11 +- .../index/query/GeoDistanceQueryBuilder.java | 9 +- .../index/mapper/DateFieldTypeTests.java | 13 +- .../index/mapper/NumberFieldTypeTests.java | 162 ++++++++++++------ .../mapper/ScaledFloatFieldTypeTests.java | 2 +- .../GeoBoundingBoxQueryBuilderTests.java | 16 ++ .../query/GeoDistanceQueryBuilderTests.java | 19 +- .../MatchPhrasePrefixQueryBuilderTests.java | 4 +- .../query/MatchPhraseQueryBuilderTests.java | 4 +- .../index/query/MatchQueryBuilderTests.java | 7 +- .../query/MultiMatchQueryBuilderTests.java | 3 +- .../index/query/RangeQueryBuilderTests.java | 15 ++ 15 files changed, 268 insertions(+), 79 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 929a3319276..f94334e3822 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -28,6 +28,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.fieldstats.FieldStats; @@ -285,7 +286,12 @@ public class DateFieldMapper extends FieldMapper { --u; } } - return LongPoint.newRangeQuery(name(), l, u); + Query query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } public long parseToMilliseconds(Object value, boolean roundUp, diff --git a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index e9043b3c754..a6482937f18 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -33,6 +33,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.PointValues; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -190,7 +191,8 @@ public class NumberFieldMapper extends FieldMapper { @Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, - boolean includeLower, boolean includeUpper) { + boolean includeLower, boolean includeUpper, + boolean hasDocValues) { float l = Float.NEGATIVE_INFINITY; float u = Float.POSITIVE_INFINITY; if (lowerTerm != null) { @@ -207,7 +209,14 @@ public class NumberFieldMapper extends FieldMapper { } u = HalfFloatPoint.nextDown(u); } - return HalfFloatPoint.newRangeQuery(field, l, u); + Query query = HalfFloatPoint.newRangeQuery(field, l, u); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, + HalfFloatPoint.halfFloatToSortableShort(l), + HalfFloatPoint.halfFloatToSortableShort(u)); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } @Override @@ -280,7 +289,8 @@ public class NumberFieldMapper extends FieldMapper { @Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, - boolean includeLower, boolean includeUpper) { + boolean includeLower, boolean includeUpper, + boolean hasDocValues) { float l = Float.NEGATIVE_INFINITY; float u = Float.POSITIVE_INFINITY; if (lowerTerm != null) { @@ -295,7 +305,14 @@ public class NumberFieldMapper extends FieldMapper { u = FloatPoint.nextDown(u); } } - return FloatPoint.newRangeQuery(field, l, u); + Query query = FloatPoint.newRangeQuery(field, l, u); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, + NumericUtils.floatToSortableInt(l), + NumericUtils.floatToSortableInt(u)); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } @Override @@ -368,7 +385,8 @@ public class NumberFieldMapper extends FieldMapper { @Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, - boolean includeLower, boolean includeUpper) { + boolean includeLower, boolean includeUpper, + boolean hasDocValues) { double l = Double.NEGATIVE_INFINITY; double u = Double.POSITIVE_INFINITY; if (lowerTerm != null) { @@ -383,7 +401,14 @@ public class NumberFieldMapper extends FieldMapper { u = DoublePoint.nextDown(u); } } - return DoublePoint.newRangeQuery(field, l, u); + Query query = DoublePoint.newRangeQuery(field, l, u); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, + NumericUtils.doubleToSortableLong(l), + NumericUtils.doubleToSortableLong(u)); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } @Override @@ -462,8 +487,9 @@ public class NumberFieldMapper extends FieldMapper { @Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, - boolean includeLower, boolean includeUpper) { - return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper); + boolean includeLower, boolean includeUpper, + boolean hasDocValues) { + return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper, hasDocValues); } @Override @@ -523,8 +549,9 @@ public class NumberFieldMapper extends FieldMapper { @Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, - boolean includeLower, boolean includeUpper) { - return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper); + boolean includeLower, boolean includeUpper, + boolean hasDocValues) { + return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper, hasDocValues); } @Override @@ -600,7 +627,8 @@ public class NumberFieldMapper extends FieldMapper { @Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, - boolean includeLower, boolean includeUpper) { + boolean includeLower, boolean includeUpper, + boolean hasDocValues) { int l = Integer.MIN_VALUE; int u = Integer.MAX_VALUE; if (lowerTerm != null) { @@ -630,7 +658,12 @@ public class NumberFieldMapper extends FieldMapper { --u; } } - return IntPoint.newRangeQuery(field, l, u); + Query query = IntPoint.newRangeQuery(field, l, u); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } @Override @@ -724,7 +757,8 @@ public class NumberFieldMapper extends FieldMapper { @Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, - boolean includeLower, boolean includeUpper) { + boolean includeLower, boolean includeUpper, + boolean hasDocValues) { long l = Long.MIN_VALUE; long u = Long.MAX_VALUE; if (lowerTerm != null) { @@ -754,7 +788,12 @@ public class NumberFieldMapper extends FieldMapper { --u; } } - return LongPoint.newRangeQuery(field, l, u); + Query query = LongPoint.newRangeQuery(field, l, u); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } @Override @@ -812,7 +851,8 @@ public class NumberFieldMapper extends FieldMapper { abstract Query termQuery(String field, Object value); abstract Query termsQuery(String field, List values); abstract Query rangeQuery(String field, Object lowerTerm, Object upperTerm, - boolean includeLower, boolean includeUpper); + boolean includeLower, boolean includeUpper, + boolean hasDocValues); abstract Number parse(XContentParser parser, boolean coerce) throws IOException; abstract Number parse(Object value, boolean coerce); public abstract List createFields(String name, Number value, boolean indexed, @@ -906,7 +946,7 @@ public class NumberFieldMapper extends FieldMapper { @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { failIfNotIndexed(); - Query query = type.rangeQuery(name(), lowerTerm, upperTerm, includeLower, includeUpper); + Query query = type.rangeQuery(name(), lowerTerm, upperTerm, includeLower, includeUpper, hasDocValues()); if (boost() != 1f) { query = new BoostQuery(query, boost()); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index cf2564f0382..8920207778e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -251,7 +251,7 @@ public class ScaledFloatFieldMapper extends FieldMapper { } hi = Math.round(Math.floor(dValue * scalingFactor)); } - Query query = NumberFieldMapper.NumberType.LONG.rangeQuery(name(), lo, hi, true, true); + Query query = NumberFieldMapper.NumberType.LONG.rangeQuery(name(), lo, hi, true, true, hasDocValues()); if (boost() != 1f) { query = new BoostQuery(query, boost()); } diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java index 74e51afac33..4045d968c5c 100644 --- a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java @@ -19,8 +19,10 @@ package org.elasticsearch.index.query; +import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.geo.Rectangle; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; @@ -348,8 +350,15 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder (byte) random().nextInt(256)); + } + + public void testDocValueShortRange() throws Exception { + doTestDocValueRangeQueries(NumberType.SHORT, () -> (short) random().nextInt(65536)); + } + + public void testDocValueIntRange() throws Exception { + doTestDocValueRangeQueries(NumberType.INTEGER, random()::nextInt); + } + + public void testDocValueLongRange() throws Exception { + doTestDocValueRangeQueries(NumberType.LONG, random()::nextLong); + } + + public void testDocValueHalfFloatRange() throws Exception { + doTestDocValueRangeQueries(NumberType.HALF_FLOAT, random()::nextFloat); + } + + public void testDocValueFloatRange() throws Exception { + doTestDocValueRangeQueries(NumberType.FLOAT, random()::nextFloat); + } + + public void testDocValueDoubleRange() throws Exception { + doTestDocValueRangeQueries(NumberType.DOUBLE, random()::nextDouble); + } + + public void doTestDocValueRangeQueries(NumberType type, Supplier valueSupplier) throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig()); + final int numDocs = TestUtil.nextInt(random(), 100, 500); + for (int i = 0; i < numDocs; ++i) { + w.addDocument(type.createFields("foo", valueSupplier.get(), true, true, false)); + } + DirectoryReader reader = DirectoryReader.open(w); + IndexSearcher searcher = newSearcher(reader); + w.close(); + final int iters = 10; + for (int iter = 0; iter < iters; ++iter) { + Query query = type.rangeQuery("foo", + random().nextBoolean() ? null : valueSupplier.get(), + random().nextBoolean() ? null : valueSupplier.get(), + randomBoolean(), randomBoolean(), true); + assertThat(query, Matchers.instanceOf(IndexOrDocValuesQuery.class)); + IndexOrDocValuesQuery indexOrDvQuery = (IndexOrDocValuesQuery) query; + assertEquals( + searcher.count(indexOrDvQuery.getIndexQuery()), + searcher.count(indexOrDvQuery.getRandomAccessQuery())); + } + reader.close(); + dir.close(); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java index 1ba58fa2ded..e1fdf103564 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java @@ -118,7 +118,7 @@ public class ScaledFloatFieldTypeTests extends FieldTypeTestCase { Double u = randomBoolean() ? null : (randomDouble() * 2 - 1) * 10000; boolean includeLower = randomBoolean(); boolean includeUpper = randomBoolean(); - Query doubleQ = NumberFieldMapper.NumberType.DOUBLE.rangeQuery("double", l, u, includeLower, includeUpper); + Query doubleQ = NumberFieldMapper.NumberType.DOUBLE.rangeQuery("double", l, u, includeLower, includeUpper, false); Query scaledFloatQ = ft.rangeQuery(l, u, includeLower, includeUpper, null); assertEquals(searcher.count(doubleQ), searcher.count(scaledFloatQ)); } diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java index 5c1b0130b43..596746938a2 100644 --- a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java @@ -19,6 +19,9 @@ package org.elasticsearch.index.query; +import org.apache.lucene.document.LatLonDocValuesField; +import org.apache.lucene.document.LatLonPoint; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.geo.GeoPoint; @@ -226,6 +229,19 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase 0); Query parsedQuery = rangeQuery(INT_FIELD_NAME).from(23).to(54).includeLower(true).includeUpper(false).toQuery(createShardContext()); // since age is automatically registered in data, we encode it as numeric + assertThat(parsedQuery, instanceOf(IndexOrDocValuesQuery.class)); + parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery(); assertThat(parsedQuery, instanceOf(PointRangeQuery.class)); assertEquals(IntPoint.newRangeQuery(INT_FIELD_NAME, 23, 53), parsedQuery); } @@ -257,6 +264,8 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase