From 0a25558f98a2d32eda61d51fef093e2394d02890 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 12 May 2017 14:33:59 +0200 Subject: [PATCH] Query range fields by doc values when they are expected to be more efficient than points. * Enable doc values for range fields by default. * Store ranges in a binary format that support multi field fields. * Added BinaryDocValuesRangeQuery that can query ranges that have been encoded into a binary doc values field. * Wrap range queries on a range field in IndexOrDocValuesQuery query. Closes #24314 --- .../queries/BinaryDocValuesRangeQuery.java | 164 +++++++++++++ .../index/mapper/BinaryRangeUtil.java | 146 ++++++++++++ .../index/mapper/RangeFieldMapper.java | 220 +++++++++++++++--- ...ndomBinaryDocValuesRangeQueryTestCase.java | 129 ++++++++++ .../BinaryDocValuesRangeQueryTests.java | 164 +++++++++++++ ...eRandomBinaryDocValuesRangeQueryTests.java | 128 ++++++++++ ...tRandomBinaryDocValuesRangeQueryTests.java | 128 ++++++++++ ...sRandomBinaryDocValuesRangeQueryTests.java | 146 ++++++++++++ ...rRandomBinaryDocValuesRangeQueryTests.java | 136 +++++++++++ ...gRandomBinaryDocValuesRangeQueryTests.java | 136 +++++++++++ .../index/mapper/BinaryRangeUtilTests.java | 97 ++++++++ .../index/mapper/RangeFieldMapperTests.java | 48 ++-- .../index/mapper/RangeFieldTypeTests.java | 99 ++++++-- 13 files changed, 1679 insertions(+), 62 deletions(-) create mode 100644 core/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java create mode 100644 core/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java create mode 100644 core/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java create mode 100644 core/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java create mode 100644 core/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java create mode 100644 core/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java create mode 100644 core/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java create mode 100644 core/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java create mode 100644 core/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java create mode 100644 core/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java diff --git a/core/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/core/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java new file mode 100644 index 00000000000..c8f78ab616d --- /dev/null +++ b/core/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java @@ -0,0 +1,164 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lucene.queries; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.util.Objects; + +public final class BinaryDocValuesRangeQuery extends Query { + + private final String fieldName; + private final QueryType queryType; + private final BytesRef from; + private final BytesRef to; + private final Object originalFrom; + private final Object originalTo; + + public BinaryDocValuesRangeQuery(String fieldName, QueryType queryType, BytesRef from, BytesRef to, + Object originalFrom, Object originalTo) { + this.fieldName = fieldName; + this.queryType = queryType; + this.from = from; + this.to = to; + this.originalFrom = originalFrom; + this.originalTo = originalTo; + } + + @Override + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + final BinaryDocValues values = context.reader().getBinaryDocValues(fieldName); + if (values == null) { + return null; + } + + final TwoPhaseIterator iterator = new TwoPhaseIterator(values) { + + ByteArrayDataInput in = new ByteArrayDataInput(); + BytesRef otherFrom = new BytesRef(16); + BytesRef otherTo = new BytesRef(16); + + @Override + public boolean matches() throws IOException { + BytesRef encodedRanges = values.binaryValue(); + in.reset(encodedRanges.bytes, encodedRanges.offset, encodedRanges.length); + int numRanges = in.readVInt(); + for (int i = 0; i < numRanges; i++) { + otherFrom.length = in.readVInt(); + otherFrom.bytes = encodedRanges.bytes; + otherFrom.offset = in.getPosition(); + in.skipBytes(otherFrom.length); + + otherTo.length = in.readVInt(); + otherTo.bytes = encodedRanges.bytes; + otherTo.offset = in.getPosition(); + in.skipBytes(otherTo.length); + + if (queryType.matches(from, to, otherFrom, otherTo)) { + return true; + } + } + return false; + } + + @Override + public float matchCost() { + return 4; // at most 4 comparisons + } + }; + return new ConstantScoreScorer(this, score(), iterator); + } + }; + } + + @Override + public String toString(String field) { + return "BinaryDocValuesRangeQuery(fieldName=" + field + ",from=" + originalFrom + ",to=" + originalTo + ")"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BinaryDocValuesRangeQuery that = (BinaryDocValuesRangeQuery) o; + return Objects.equals(fieldName, that.fieldName) && + queryType == that.queryType && + Objects.equals(from, that.from) && + Objects.equals(to, that.to); + } + + @Override + public int hashCode() { + return Objects.hash(getClass(), fieldName, queryType, from, to); + } + + public enum QueryType { + INTERSECTS { + @Override + boolean matches(BytesRef from, BytesRef to, BytesRef otherFrom, BytesRef otherTo) { + // part of the other range must touch this range + // this: |---------------| + // other: |------| + return from.compareTo(otherTo) <= 0 && to.compareTo(otherFrom) >= 0; + } + }, WITHIN { + @Override + boolean matches(BytesRef from, BytesRef to, BytesRef otherFrom, BytesRef otherTo) { + // other range must entirely lie within this range + // this: |---------------| + // other: |------| + return from.compareTo(otherFrom) <= 0 && to.compareTo(otherTo) >= 0; + } + }, CONTAINS { + @Override + boolean matches(BytesRef from, BytesRef to, BytesRef otherFrom, BytesRef otherTo) { + // this and other range must overlap + // this: |------| + // other: |---------------| + return from.compareTo(otherFrom) >= 0 && to.compareTo(otherTo) <= 0; + } + }, CROSSES { + @Override + boolean matches(BytesRef from, BytesRef to, BytesRef otherFrom, BytesRef otherTo) { + // does not disjoint AND not within: + return (from.compareTo(otherTo) > 0 || to.compareTo(otherFrom) < 0) == false && + (from.compareTo(otherFrom) <= 0 && to.compareTo(otherTo) >= 0) == false; + } + }; + + abstract boolean matches(BytesRef from, BytesRef to, BytesRef otherFrom, BytesRef otherTo); + + } + +} diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java b/core/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java new file mode 100644 index 00000000000..e2b618bc222 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/mapper/BinaryRangeUtil.java @@ -0,0 +1,146 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +enum BinaryRangeUtil { + + ; + + static BytesRef encodeLongRanges(Set ranges) throws IOException { + List sortedRanges = new ArrayList<>(ranges); + sortedRanges.sort((r1, r2) -> { + long r1From = ((Number) r1.from).longValue(); + long r2From = ((Number) r2.from).longValue(); + int cmp = Long.compare(r1From, r2From); + if (cmp != 0) { + return cmp; + } else { + long r1To = ((Number) r1.from).longValue(); + long r2To = ((Number) r2.from).longValue(); + return Long.compare(r1To, r2To); + } + }); + + final byte[] encoded = new byte[5 + ((5 + 9) * 2) * sortedRanges.size()]; + ByteArrayDataOutput out = new ByteArrayDataOutput(encoded); + out.writeVInt(sortedRanges.size()); + for (RangeFieldMapper.Range range : sortedRanges) { + byte[] encodedFrom = encode(((Number) range.from).longValue()); + out.writeVInt(encodedFrom.length); + out.writeBytes(encodedFrom, encodedFrom.length); + byte[] encodedTo = encode(((Number) range.to).longValue()); + out.writeVInt(encodedTo.length); + out.writeBytes(encodedTo, encodedTo.length); + } + return new BytesRef(encoded, 0, out.getPosition()); + } + + static BytesRef encodeDoubleRanges(Set ranges) throws IOException { + List sortedRanges = new ArrayList<>(ranges); + sortedRanges.sort((r1, r2) -> { + double r1From = ((Number) r1.from).doubleValue(); + double r2From = ((Number) r2.from).doubleValue(); + int cmp = Double.compare(r1From, r2From); + if (cmp != 0) { + return cmp; + } else { + double r1To = ((Number) r1.from).doubleValue(); + double r2To = ((Number) r2.from).doubleValue(); + return Double.compare(r1To, r2To); + } + }); + + final byte[] encoded = new byte[5 + ((5 + 9) * 2) * sortedRanges.size()]; + ByteArrayDataOutput out = new ByteArrayDataOutput(encoded); + out.writeVInt(sortedRanges.size()); + for (RangeFieldMapper.Range range : sortedRanges) { + byte[] encodedFrom = BinaryRangeUtil.encode(((Number) range.from).doubleValue()); + out.writeVInt(encodedFrom.length); + out.writeBytes(encodedFrom, encodedFrom.length); + byte[] encodedTo = BinaryRangeUtil.encode(((Number) range.to).doubleValue()); + out.writeVInt(encodedTo.length); + out.writeBytes(encodedTo, encodedTo.length); + } + return new BytesRef(encoded, 0, out.getPosition()); + } + + /** + * Encodes the specified number of type long in a variable-length byte format. + * The byte format preserves ordering, which means the returned byte array can be used for comparing as is. + */ + static byte[] encode(long number) { + int sign = 1; // means positive + if (number < 0) { + number = -1 - number; + sign = 0; + } + return encode(number, sign); + } + + /** + * Encodes the specified number of type double in a variable-length byte format. + * The byte format preserves ordering, which means the returned byte array can be used for comparing as is. + */ + static byte[] encode(double number) { + long l; + int sign; + if (number < 0.0) { + l = Double.doubleToRawLongBits(-0d - number); + sign = 0; + } else { + l = Double.doubleToRawLongBits(number); + sign = 1; // means positive + } + return encode(l, sign); + } + + private static byte[] encode(long l, int sign) { + assert l >= 0; + int bits = 64 - Long.numberOfLeadingZeros(l); + + int numBytes = (bits + 7) / 8; // between 0 and 8 + byte[] encoded = new byte[1 + numBytes]; + // encode the sign first to make sure positive values compare greater than negative values + // and then the number of bytes, to make sure that large values compare greater than low values + if (sign > 0) { + encoded[0] = (byte) ((sign << 4) | numBytes); + } else { + encoded[0] = (byte) ((sign << 4) | (8 - numBytes)); + } + for (int b = 0; b < numBytes; ++b) { + if (sign == 1) { + encoded[encoded.length - 1 - b] = (byte) (l >>> (8 * b)); + } else if (sign == 0) { + encoded[encoded.length - 1 - b] = (byte) (0xFF - ((l >>> (8 * b)) & 0xFF)); + } else { + throw new AssertionError(); + } + } + return encoded; + } + +} diff --git a/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 38c799ee3f6..d2a6f28169f 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -18,19 +18,25 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.Field; import org.apache.lucene.document.DoubleRange; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatRange; -import org.apache.lucene.document.IntRange; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.InetAddressRange; +import org.apache.lucene.document.IntRange; import org.apache.lucene.document.LongRange; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queries.BinaryDocValuesRangeQuery; +import org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.geo.ShapeRelation; @@ -49,17 +55,19 @@ import org.joda.time.DateTimeZone; import java.io.IOException; import java.net.InetAddress; import java.util.ArrayList; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter; -import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; -import static org.elasticsearch.index.query.RangeQueryBuilder.LT_FIELD; +import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD; import static org.elasticsearch.index.query.RangeQueryBuilder.LTE_FIELD; +import static org.elasticsearch.index.query.RangeQueryBuilder.LT_FIELD; /** A {@link FieldMapper} for indexing numeric and date ranges, and creating queries */ public class RangeFieldMapper extends FieldMapper { @@ -78,8 +86,8 @@ public class RangeFieldMapper extends FieldMapper { private Boolean coerce; private Locale locale; - public Builder(String name, RangeType type) { - super(name, new RangeFieldType(type), new RangeFieldType(type)); + public Builder(String name, RangeType type, Version indexVersionCreated) { + super(name, new RangeFieldType(type, indexVersionCreated), new RangeFieldType(type, indexVersionCreated)); builder = this; locale = Locale.ROOT; } @@ -159,7 +167,7 @@ public class RangeFieldMapper extends FieldMapper { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = new Builder(name, type); + Builder builder = new Builder(name, type, parserContext.indexVersionCreated()); TypeParsers.parseField(builder, name, node, parserContext); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -190,18 +198,18 @@ public class RangeFieldMapper extends FieldMapper { protected FormatDateTimeFormatter dateTimeFormatter; protected DateMathParser dateMathParser; - public RangeFieldType(RangeType type) { + RangeFieldType(RangeType type, Version indexVersionCreated) { super(); this.rangeType = Objects.requireNonNull(type); setTokenized(false); - setHasDocValues(false); + setHasDocValues(indexVersionCreated.onOrAfter(Version.V_6_0_0_beta1)); setOmitNorms(true); if (rangeType == RangeType.DATE) { setDateTimeFormatter(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER); } } - public RangeFieldType(RangeFieldType other) { + RangeFieldType(RangeFieldType other) { super(other); this.rangeType = other.rangeType; if (other.dateTimeFormatter() != null) { @@ -290,7 +298,8 @@ public class RangeFieldMapper extends FieldMapper { public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, ShapeRelation relation, DateTimeZone timeZone, DateMathParser parser, QueryShardContext context) { - return rangeType.rangeQuery(name(), lowerTerm, upperTerm, includeLower, includeUpper, relation, timeZone, parser, context); + return rangeType.rangeQuery(name(), hasDocValues(), lowerTerm, upperTerm, includeLower, includeUpper, relation, + timeZone, parser, context); } } @@ -385,7 +394,7 @@ public class RangeFieldMapper extends FieldMapper { boolean indexed = fieldType.indexOptions() != IndexOptions.NONE; boolean docValued = fieldType.hasDocValues(); boolean stored = fieldType.stored(); - fields.addAll(fieldType().rangeType.createFields(name(), range, indexed, docValued, stored)); + fields.addAll(fieldType().rangeType.createFields(context, name(), range, indexed, docValued, stored)); } @Override @@ -461,6 +470,33 @@ public class RangeFieldMapper extends FieldMapper { public InetAddress nextDown(Object value) { return InetAddressPoint.nextDown((InetAddress)value); } + + @Override + public BytesRef encodeRanges(Set ranges) throws IOException { + final byte[] encoded = new byte[5 + (16 * 2) * ranges.size()]; + ByteArrayDataOutput out = new ByteArrayDataOutput(encoded); + out.writeVInt(ranges.size()); + for (Range range : ranges) { + out.writeVInt(16); + InetAddress fromValue = (InetAddress) range.from; + byte[] encodedFromValue = InetAddressPoint.encode(fromValue); + out.writeBytes(encodedFromValue, 0, encodedFromValue.length); + + out.writeVInt(16); + InetAddress toValue = (InetAddress) range.to; + byte[] encodedToValue = InetAddressPoint.encode(toValue); + out.writeBytes(encodedToValue, 0, encodedToValue.length); + } + return new BytesRef(encoded, 0, out.getPosition()); + } + + @Override + BytesRef[] encodeRange(Object from, Object to) { + BytesRef encodedFrom = new BytesRef(InetAddressPoint.encode((InetAddress) from)); + BytesRef encodedTo = new BytesRef(InetAddressPoint.encode((InetAddress) to)); + return new BytesRef[]{encodedFrom, encodedTo}; + } + @Override public Query withinQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) { InetAddress lower = (InetAddress)from; @@ -522,10 +558,21 @@ public class RangeFieldMapper extends FieldMapper { public Long nextDown(Object value) { return (long) LONG.nextDown(value); } + @Override - public Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, - ShapeRelation relation, @Nullable DateTimeZone timeZone, @Nullable DateMathParser parser, - QueryShardContext context) { + public BytesRef encodeRanges(Set ranges) throws IOException { + return LONG.encodeRanges(ranges); + } + + @Override + BytesRef[] encodeRange(Object from, Object to) { + return LONG.encodeRange(from, to); + } + + @Override + public Query rangeQuery(String field, boolean hasDocValues, Object lowerTerm, Object upperTerm, boolean includeLower, + boolean includeUpper, ShapeRelation relation, @Nullable DateTimeZone timeZone, + @Nullable DateMathParser parser, QueryShardContext context) { DateTimeZone zone = (timeZone == null) ? DateTimeZone.UTC : timeZone; DateMathParser dateMathParser = (parser == null) ? new DateMathParser(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER) : parser; @@ -536,7 +583,8 @@ public class RangeFieldMapper extends FieldMapper { dateMathParser.parse(upperTerm instanceof BytesRef ? ((BytesRef) upperTerm).utf8ToString() : upperTerm.toString(), context::nowInMillis, false, zone); - return super.rangeQuery(field, low, high, includeLower, includeUpper, relation, zone, dateMathParser, context); + return super.rangeQuery(field, hasDocValues, low, high, includeLower, includeUpper, relation, zone, + dateMathParser, context); } @Override public Query withinQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) { @@ -569,6 +617,17 @@ public class RangeFieldMapper extends FieldMapper { public Float nextDown(Object value) { return Math.nextDown(((Number)value).floatValue()); } + + @Override + public BytesRef encodeRanges(Set ranges) throws IOException { + return DOUBLE.encodeRanges(ranges); + } + + @Override + BytesRef[] encodeRange(Object from, Object to) { + return DOUBLE.encodeRange(((Number) from).floatValue(), ((Number) to).floatValue()); + } + @Override public Field getRangeField(String name, Range r) { return new FloatRange(name, new float[] {((Number)r.from).floatValue()}, new float[] {((Number)r.to).floatValue()}); @@ -609,6 +668,19 @@ public class RangeFieldMapper extends FieldMapper { public Double nextDown(Object value) { return Math.nextDown(((Number)value).doubleValue()); } + + @Override + public BytesRef encodeRanges(Set ranges) throws IOException { + return BinaryRangeUtil.encodeDoubleRanges(ranges); + } + + @Override + BytesRef[] encodeRange(Object from, Object to) { + byte[] fromValue = BinaryRangeUtil.encode(((Number) from).doubleValue()); + byte[] toValue = BinaryRangeUtil.encode(((Number) to).doubleValue()); + return new BytesRef[]{new BytesRef(fromValue), new BytesRef(toValue)}; + } + @Override public Field getRangeField(String name, Range r) { return new DoubleRange(name, new double[] {((Number)r.from).doubleValue()}, new double[] {((Number)r.to).doubleValue()}); @@ -651,6 +723,17 @@ public class RangeFieldMapper extends FieldMapper { public Integer nextDown(Object value) { return ((Number)value).intValue() - 1; } + + @Override + public BytesRef encodeRanges(Set ranges) throws IOException { + return LONG.encodeRanges(ranges); + } + + @Override + BytesRef[] encodeRange(Object from, Object to) { + return LONG.encodeRange(from, to); + } + @Override public Field getRangeField(String name, Range r) { return new IntRange(name, new int[] {((Number)r.from).intValue()}, new int[] {((Number)r.to).intValue()}); @@ -688,6 +771,19 @@ public class RangeFieldMapper extends FieldMapper { public Long nextDown(Object value) { return ((Number)value).longValue() - 1; } + + @Override + public BytesRef encodeRanges(Set ranges) throws IOException { + return BinaryRangeUtil.encodeLongRanges(ranges); + } + + @Override + BytesRef[] encodeRange(Object from, Object to) { + byte[] encodedFrom = BinaryRangeUtil.encode(((Number) from).longValue()); + byte[] encodedTo = BinaryRangeUtil.encode(((Number) to).longValue()); + return new BytesRef[]{new BytesRef(encodedFrom), new BytesRef(encodedTo)}; + } + @Override public Field getRangeField(String name, Range r) { return new LongRange(name, new long[] {((Number)r.from).longValue()}, @@ -726,13 +822,22 @@ public class RangeFieldMapper extends FieldMapper { } public abstract Field getRangeField(String name, Range range); - public List createFields(String name, Range range, boolean indexed, boolean docValued, boolean stored) { + public List createFields(ParseContext context, String name, Range range, boolean indexed, + boolean docValued, boolean stored) { assert range != null : "range cannot be null when creating fields"; List fields = new ArrayList<>(); if (indexed) { fields.add(getRangeField(name, range)); } - // todo add docValues ranges once aggregations are supported + if (docValued) { + BinaryRangesDocValuesField field = (BinaryRangesDocValuesField) context.doc().getByKey(name); + if (field == null) { + field = new BinaryRangesDocValuesField(name, range, this); + context.doc().addWithKey(name, field); + } else { + field.add(range); + } + } if (stored) { fields.add(new StoredField(name, range.toString())); } @@ -759,28 +864,65 @@ public class RangeFieldMapper extends FieldMapper { public Object parse(Object value, boolean coerce) { return numberType.parse(value, coerce); } - public Query rangeQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo, - ShapeRelation relation, @Nullable DateTimeZone timeZone, @Nullable DateMathParser dateMathParser, - QueryShardContext context) { + public Query rangeQuery(String field, boolean hasDocValues, Object from, Object to, boolean includeFrom, boolean includeTo, + ShapeRelation relation, @Nullable DateTimeZone timeZone, @Nullable DateMathParser dateMathParser, + QueryShardContext context) { Object lower = from == null ? minValue() : parse(from, false); Object upper = to == null ? maxValue() : parse(to, false); + Query indexQuery; if (relation == ShapeRelation.WITHIN) { - return withinQuery(field, lower, upper, includeFrom, includeTo); + indexQuery = withinQuery(field, lower, upper, includeFrom, includeTo); } else if (relation == ShapeRelation.CONTAINS) { - return containsQuery(field, lower, upper, includeFrom, includeTo); + indexQuery = containsQuery(field, lower, upper, includeFrom, includeTo); + } else { + indexQuery = intersectsQuery(field, lower, upper, includeFrom, includeTo); + } + if (hasDocValues) { + final QueryType queryType; + if (relation == ShapeRelation.WITHIN) { + queryType = QueryType.WITHIN; + } else if (relation == ShapeRelation.CONTAINS) { + queryType = QueryType.CONTAINS; + } else { + queryType = QueryType.INTERSECTS; + } + Query dvQuery = dvRangeQuery(field, queryType, lower, upper, includeFrom, includeTo); + return new IndexOrDocValuesQuery(indexQuery, dvQuery); + } else { + return indexQuery; } - return intersectsQuery(field, lower, upper, includeFrom, includeTo); } + // No need to take into account Range#includeFrom or Range#includeTo, because from and to have already been + // rounded up via parseFrom and parseTo methods. + public abstract BytesRef encodeRanges(Set ranges) throws IOException; + + public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) { + if (includeFrom == false) { + from = nextUp(from); + } + + if (includeTo == false) { + to = nextDown(to); + } + BytesRef[] range = encodeRange(from, to); + return new BinaryDocValuesRangeQuery(field, queryType, range[0], range[1], from, to); + } + + abstract BytesRef[] encodeRange(Object from, Object to); + public final String name; private final NumberType numberType; + + + } /** Class defining a range */ public static class Range { RangeType type; - private Object from; - private Object to; + Object from; + Object to; private boolean includeFrom; private boolean includeTo; @@ -805,4 +947,30 @@ public class RangeFieldMapper extends FieldMapper { return sb.toString(); } } + + static class BinaryRangesDocValuesField extends CustomDocValuesField { + + private final Set ranges; + private final RangeType rangeType; + + BinaryRangesDocValuesField(String name, Range range, RangeType rangeType) { + super(name); + this.rangeType = rangeType; + ranges = new HashSet<>(); + add(range); + } + + void add(Range range) { + ranges.add(range); + } + + @Override + public BytesRef binaryValue() { + try { + return rangeType.encodeRanges(ranges); + } catch (IOException e) { + throw new ElasticsearchException("failed to encode ranges", e); + } + } + } } diff --git a/core/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java b/core/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java new file mode 100644 index 00000000000..b83dac78d07 --- /dev/null +++ b/core/src/test/java/org/apache/lucene/queries/BaseRandomBinaryDocValuesRangeQueryTestCase.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lucene.queries; + +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.Field; +import org.apache.lucene.search.BaseRangeFieldQueryTestCase; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.RangeFieldMapper; + +import java.io.IOException; +import java.util.Collections; +import java.util.Objects; + +import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CONTAINS; +import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CROSSES; +import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.INTERSECTS; +import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.WITHIN; + +public abstract class BaseRandomBinaryDocValuesRangeQueryTestCase extends BaseRangeFieldQueryTestCase { + + @Override + public void testMultiValued() throws Exception { + // Can't test this how BaseRangeFieldQueryTestCase works now, because we're using BinaryDocValuesField here. + } + + @Override + public void testRandomBig() throws Exception { + // Test regardless whether -Dtests.nightly=true has been specified: + super.testRandomBig(); + } + + @Override + protected final Field newRangeField(Range box) { + AbstractRange testRange = (AbstractRange) box; + RangeFieldMapper.Range range = new RangeFieldMapper.Range(rangeType(), testRange.getMin(), testRange.getMax(), true , true); + try { + BytesRef encodeRange = rangeType().encodeRanges(Collections.singleton(range)); + return new BinaryDocValuesField(fieldName(), encodeRange); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + protected final Query newIntersectsQuery(Range box) { + AbstractRange testRange = (AbstractRange) box; + return rangeType().dvRangeQuery(fieldName(), INTERSECTS, testRange.getMin(), testRange.getMax(), true, true); + } + + @Override + protected final Query newContainsQuery(Range box) { + AbstractRange testRange = (AbstractRange) box; + return rangeType().dvRangeQuery(fieldName(), CONTAINS, testRange.getMin(), testRange.getMax(), true, true); + } + + @Override + protected final Query newWithinQuery(Range box) { + AbstractRange testRange = (AbstractRange) box; + return rangeType().dvRangeQuery(fieldName(), WITHIN, testRange.getMin(), testRange.getMax(), true, true); + } + + @Override + protected final Query newCrossesQuery(Range box) { + AbstractRange testRange = (AbstractRange) box; + return rangeType().dvRangeQuery(fieldName(), CROSSES, testRange.getMin(), testRange.getMax(), true, true); + } + + @Override + protected final int dimension() { + return 1; + } + + protected abstract String fieldName(); + + protected abstract RangeFieldMapper.RangeType rangeType(); + + protected abstract static class AbstractRange extends Range { + + protected final int numDimensions() { + return 1; + } + + @Override + protected final Object getMin(int dim) { + assert dim == 0; + return getMin(); + } + + public abstract T getMin(); + + @Override + protected final Object getMax(int dim) { + assert dim == 0; + return getMax(); + } + + public abstract T getMax(); + + @Override + protected final boolean isEqual(Range o) { + AbstractRange other = (AbstractRange) o; + return Objects.equals(getMin(), other.getMin()) && Objects.equals(getMax(), other.getMax()); + } + + @Override + public final String toString() { + return "Box(" + getMin() + " TO " + getMax() + ")"; + } + } + +} diff --git a/core/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java new file mode 100644 index 00000000000..921d1ed5f1f --- /dev/null +++ b/core/src/test/java/org/apache/lucene/queries/BinaryDocValuesRangeQueryTests.java @@ -0,0 +1,164 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lucene.queries; + +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.RangeFieldMapper; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static java.util.Collections.singleton; +import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CONTAINS; +import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.CROSSES; +import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.INTERSECTS; +import static org.apache.lucene.queries.BinaryDocValuesRangeQuery.QueryType.WITHIN; + +public class BinaryDocValuesRangeQueryTests extends ESTestCase { + + public void testBasics() throws Exception { + String fieldName = "long_field"; + RangeFieldMapper.RangeType rangeType = RangeFieldMapper.RangeType.LONG; + try (Directory dir = newDirectory()) { + try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) { + // intersects (within) + Document document = new Document(); + BytesRef encodedRange = + rangeType.encodeRanges(singleton(new RangeFieldMapper.Range(rangeType, -10L, 9L, true , true))); + document.add(new BinaryDocValuesField(fieldName, encodedRange)); + writer.addDocument(document); + + // intersects (crosses) + document = new Document(); + encodedRange = rangeType.encodeRanges(singleton(new RangeFieldMapper.Range(rangeType, 10L, 20L, true , true))); + document.add(new BinaryDocValuesField(fieldName, encodedRange)); + writer.addDocument(document); + + // intersects (contains, crosses) + document = new Document(); + encodedRange = rangeType.encodeRanges(singleton(new RangeFieldMapper.Range(rangeType, -20L, 30L, true , true))); + document.add(new BinaryDocValuesField(fieldName, encodedRange)); + writer.addDocument(document); + + // intersects (within) + document = new Document(); + encodedRange = rangeType.encodeRanges(singleton(new RangeFieldMapper.Range(rangeType, -11L, 1L, true , true))); + document.add(new BinaryDocValuesField(fieldName, encodedRange)); + writer.addDocument(document); + + // intersects (crosses) + document = new Document(); + encodedRange = rangeType.encodeRanges(singleton(new RangeFieldMapper.Range(rangeType, 12L, 15L, true , true))); + document.add(new BinaryDocValuesField(fieldName, encodedRange)); + writer.addDocument(document); + + // disjoint + document = new Document(); + encodedRange = rangeType.encodeRanges(singleton(new RangeFieldMapper.Range(rangeType, -122L, -115L, true , true))); + document.add(new BinaryDocValuesField(fieldName, encodedRange)); + writer.addDocument(document); + + // intersects (crosses) + document = new Document(); + encodedRange = rangeType.encodeRanges(singleton(new RangeFieldMapper.Range(rangeType, Long.MIN_VALUE, -11L, true , true))); + document.add(new BinaryDocValuesField(fieldName, encodedRange)); + writer.addDocument(document); + + // equal (within, contains, intersects) + document = new Document(); + encodedRange = rangeType.encodeRanges(singleton(new RangeFieldMapper.Range(rangeType, -11L, 15L, true , true))); + document.add(new BinaryDocValuesField(fieldName, encodedRange)); + writer.addDocument(document); + + // intersects, within + document = new Document(); + encodedRange = rangeType.encodeRanges(singleton(new RangeFieldMapper.Range(rangeType, 5L, 10L, true , true))); + document.add(new BinaryDocValuesField(fieldName, encodedRange)); + writer.addDocument(document); + + // search + try (IndexReader reader = writer.getReader()) { + IndexSearcher searcher = newSearcher(reader); + Query query = rangeType.dvRangeQuery(fieldName, INTERSECTS, -11L, 15L, true, true); + assertEquals(8, searcher.count(query)); + query = rangeType.dvRangeQuery(fieldName, WITHIN, -11L, 15L, true, true); + assertEquals(5, searcher.count(query)); + query = rangeType.dvRangeQuery(fieldName, CONTAINS, -11L, 15L, true, true); + assertEquals(2, searcher.count(query)); + query = rangeType.dvRangeQuery(fieldName, CROSSES, -11L, 15L, true, true); + assertEquals(3, searcher.count(query)); + + // test includeFrom = false and includeTo = false + query = rangeType.dvRangeQuery(fieldName, INTERSECTS, -11L, 15L, false, false); + assertEquals(7, searcher.count(query)); + query = rangeType.dvRangeQuery(fieldName, WITHIN, -11L, 15L, false, false); + assertEquals(2, searcher.count(query)); + query = rangeType.dvRangeQuery(fieldName, CONTAINS, -11L, 15L, false, false); + assertEquals(2, searcher.count(query)); + query = rangeType.dvRangeQuery(fieldName, CROSSES, -11L, 15L, false, false); + assertEquals(5, searcher.count(query)); + } + } + } + } + + public void testNoField() throws IOException { + String fieldName = "long_field"; + RangeFieldMapper.RangeType rangeType = RangeFieldMapper.RangeType.LONG; + + // no field in index + try (Directory dir = newDirectory()) { + try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) { + writer.addDocument(new Document()); + try (IndexReader reader = writer.getReader()) { + IndexSearcher searcher = newSearcher(reader); + Query query = rangeType.dvRangeQuery(fieldName, INTERSECTS, -1L, 1L, true, true); + assertEquals(0, searcher.count(query)); + } + } + } + + // no field in segment + try (Directory dir = newDirectory()) { + try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) { + // intersects (within) + Document document = new Document(); + BytesRef encodedRange = + rangeType.encodeRanges(singleton(new RangeFieldMapper.Range(rangeType, 0L, 0L, true , true))); + document.add(new BinaryDocValuesField(fieldName, encodedRange)); + writer.addDocument(document); + writer.commit(); + writer.addDocument(new Document()); + try (IndexReader reader = writer.getReader()) { + IndexSearcher searcher = newSearcher(reader); + Query query = rangeType.dvRangeQuery(fieldName, INTERSECTS, -1L, 1L, true, true); + assertEquals(1, searcher.count(query)); + } + } + } + } + +} diff --git a/core/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java new file mode 100644 index 00000000000..aa15a803195 --- /dev/null +++ b/core/src/test/java/org/apache/lucene/queries/DoubleRandomBinaryDocValuesRangeQueryTests.java @@ -0,0 +1,128 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lucene.queries; + +import org.elasticsearch.index.mapper.RangeFieldMapper; + +public class DoubleRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { + + @Override + protected String fieldName() { + return "double_range_dv_field"; + } + + @Override + protected RangeFieldMapper.RangeType rangeType() { + return RangeFieldMapper.RangeType.DOUBLE; + } + + @Override + protected Range nextRange(int dimensions) throws Exception { + double value1 = nextDoubleInternal(); + double value2 = nextDoubleInternal(); + double min = Math.min(value1, value2); + double max = Math.max(value1, value2); + return new DoubleTestRange(min, max); + } + + private double nextDoubleInternal() { + switch (random().nextInt(5)) { + case 0: + return Double.NEGATIVE_INFINITY; + case 1: + return Double.POSITIVE_INFINITY; + default: + if (random().nextBoolean()) { + return random().nextDouble(); + } else { + return (random().nextInt(15) - 7) / 3d; + } + } + } + + private static class DoubleTestRange extends AbstractRange { + double min; + double max; + + DoubleTestRange(double min, double max) { + this.min = min; + this.max = max; + } + + @Override + public Object getMin() { + return min; + } + + @Override + protected void setMin(int dim, Object val) { + assert dim == 0; + double v = (Double) val; + if (min < v) { + max = v; + } else { + min = v; + } + } + + @Override + public Object getMax() { + return max; + } + + @Override + protected void setMax(int dim, Object val) { + assert dim == 0; + double v = (Double) val; + if (max > v) { + min = v; + } else { + max = v; + } + } + + @Override + protected boolean isDisjoint(Range o) { + DoubleTestRange other = (DoubleTestRange)o; + return this.min > other.max || this.max < other.min; + } + + @Override + protected boolean isWithin(Range o) { + DoubleTestRange other = (DoubleTestRange)o; + if ((this.min >= other.min && this.max <= other.max) == false) { + // not within: + return false; + } + return true; + } + + @Override + protected boolean contains(Range o) { + DoubleTestRange other = (DoubleTestRange) o; + if ((this.min <= other.min && this.max >= other.max) == false) { + // not contains: + return false; + } + return true; + } + + } + +} diff --git a/core/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java new file mode 100644 index 00000000000..8a04a50448f --- /dev/null +++ b/core/src/test/java/org/apache/lucene/queries/FloatRandomBinaryDocValuesRangeQueryTests.java @@ -0,0 +1,128 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lucene.queries; + +import org.elasticsearch.index.mapper.RangeFieldMapper; + +public class FloatRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { + + @Override + protected String fieldName() { + return "float_range_dv_field"; + } + + @Override + protected RangeFieldMapper.RangeType rangeType() { + return RangeFieldMapper.RangeType.FLOAT; + } + + @Override + protected Range nextRange(int dimensions) throws Exception { + float value1 = nextFloatInternal(); + float value2 = nextFloatInternal(); + float min = Math.min(value1, value2); + float max = Math.max(value1, value2); + return new FloatTestRange(min, max); + } + + private float nextFloatInternal() { + switch (random().nextInt(5)) { + case 0: + return Float.NEGATIVE_INFINITY; + case 1: + return Float.POSITIVE_INFINITY; + default: + if (random().nextBoolean()) { + return random().nextFloat(); + } else { + return (random().nextInt(15) - 7) / 3f; + } + } + } + + private static class FloatTestRange extends AbstractRange { + float min; + float max; + + FloatTestRange(float min, float max) { + this.min = min; + this.max = max; + } + + @Override + public Object getMin() { + return min; + } + + @Override + protected void setMin(int dim, Object val) { + assert dim == 0; + float v = (Float) val; + if (min < v) { + max = v; + } else { + min = v; + } + } + + @Override + public Object getMax() { + return max; + } + + @Override + protected void setMax(int dim, Object val) { + assert dim == 0; + float v = (Float) val; + if (max > v) { + min = v; + } else { + max = v; + } + } + + @Override + protected boolean isDisjoint(Range o) { + FloatTestRange other = (FloatTestRange)o; + return this.min > other.max || this.max < other.min; + } + + @Override + protected boolean isWithin(Range o) { + FloatTestRange other = (FloatTestRange)o; + if ((this.min >= other.min && this.max <= other.max) == false) { + // not within: + return false; + } + return true; + } + + @Override + protected boolean contains(Range o) { + FloatTestRange other = (FloatTestRange) o; + if ((this.min <= other.min && this.max >= other.max) == false) { + // not contains: + return false; + } + return true; + } + + } + +} diff --git a/core/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java new file mode 100644 index 00000000000..1592e89d174 --- /dev/null +++ b/core/src/test/java/org/apache/lucene/queries/InetAddressRandomBinaryDocValuesRangeQueryTests.java @@ -0,0 +1,146 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lucene.queries; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.StringHelper; +import org.elasticsearch.index.mapper.RangeFieldMapper; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; + +public class InetAddressRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { + + @Override + protected String fieldName() { + return "ip_range_dv_field"; + } + + @Override + protected RangeFieldMapper.RangeType rangeType() { + return RangeFieldMapper.RangeType.IP; + } + + @Override + protected Range nextRange(int dimensions) throws Exception { + InetAddress min = nextInetaddress(); + byte[] bMin = InetAddressPoint.encode(min); + InetAddress max = nextInetaddress(); + byte[] bMax = InetAddressPoint.encode(max); + if (StringHelper.compare(bMin.length, bMin, 0, bMax, 0) > 0) { + return new IpRange(max, min); + } + return new IpRange(min, max); + } + + private InetAddress nextInetaddress() throws UnknownHostException { + byte[] b = random().nextBoolean() ? new byte[4] : new byte[16]; + switch (random().nextInt(5)) { + case 0: + return InetAddress.getByAddress(b); + case 1: + Arrays.fill(b, (byte) 0xff); + return InetAddress.getByAddress(b); + case 2: + Arrays.fill(b, (byte) 42); + return InetAddress.getByAddress(b); + default: + random().nextBytes(b); + return InetAddress.getByAddress(b); + } + } + + private static class IpRange extends AbstractRange { + InetAddress minAddress; + InetAddress maxAddress; + byte[] min; + byte[] max; + + IpRange(InetAddress min, InetAddress max) { + this.minAddress = min; + this.maxAddress = max; + this.min = InetAddressPoint.encode(min); + this.max = InetAddressPoint.encode(max); + } + + @Override + public Object getMin() { + return minAddress; + } + + @Override + protected void setMin(int dim, Object val) { + assert dim == 0; + InetAddress v = (InetAddress)val; + byte[] e = InetAddressPoint.encode(v); + + if (StringHelper.compare(e.length, min, 0, e, 0) < 0) { + max = e; + maxAddress = v; + } else { + min = e; + minAddress = v; + } + } + + @Override + public Object getMax() { + return maxAddress; + } + + @Override + protected void setMax(int dim, Object val) { + assert dim == 0; + InetAddress v = (InetAddress)val; + byte[] e = InetAddressPoint.encode(v); + + if (StringHelper.compare(e.length, max, 0, e, 0) > 0) { + min = e; + minAddress = v; + } else { + max = e; + maxAddress = v; + } + } + + @Override + protected boolean isDisjoint(Range o) { + IpRange other = (IpRange) o; + return StringHelper.compare(min.length, min, 0, other.max, 0) > 0 || + StringHelper.compare(max.length, max, 0, other.min, 0) < 0; + } + + @Override + protected boolean isWithin(Range o) { + IpRange other = (IpRange)o; + return StringHelper.compare(min.length, min, 0, other.min, 0) >= 0 && + StringHelper.compare(max.length, max, 0, other.max, 0) <= 0; + } + + @Override + protected boolean contains(Range o) { + IpRange other = (IpRange)o; + return StringHelper.compare(min.length, min, 0, other.min, 0) <= 0 && + StringHelper.compare(max.length, max, 0, other.max, 0) >= 0; + } + + } + +} diff --git a/core/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java new file mode 100644 index 00000000000..6fe59b88273 --- /dev/null +++ b/core/src/test/java/org/apache/lucene/queries/IntegerRandomBinaryDocValuesRangeQueryTests.java @@ -0,0 +1,136 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lucene.queries; + +import org.apache.lucene.util.TestUtil; +import org.elasticsearch.index.mapper.RangeFieldMapper; + +public class IntegerRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { + + @Override + protected String fieldName() { + return "int_range_dv_field"; + } + + @Override + protected RangeFieldMapper.RangeType rangeType() { + return RangeFieldMapper.RangeType.INTEGER; + } + + @Override + protected Range nextRange(int dimensions) throws Exception { + int value1 = nextIntInternal(); + int value2 = nextIntInternal(); + int min = Math.min(value1, value2); + int max = Math.max(value1, value2); + return new IntTestRange(min, max); + } + + private int nextIntInternal() { + switch (random().nextInt(5)) { + case 0: + return Integer.MIN_VALUE; + case 1: + return Integer.MAX_VALUE; + default: + int bpv = random().nextInt(32); + switch (bpv) { + case 32: + return random().nextInt(); + default: + int v = TestUtil.nextInt(random(), 0, (1 << bpv) - 1); + if (bpv > 0) { + // negative values sometimes + v -= 1 << (bpv - 1); + } + return v; + } + } + } + + private static class IntTestRange extends AbstractRange { + int min; + int max; + + IntTestRange(int min, int max) { + this.min = min; + this.max = max; + } + + @Override + public Object getMin() { + return min; + } + + @Override + protected void setMin(int dim, Object val) { + assert dim == 0; + int v = (Integer) val; + if (min < v) { + max = v; + } else { + min = v; + } + } + + @Override + public Object getMax() { + return max; + } + + @Override + protected void setMax(int dim, Object val) { + assert dim == 0; + int v = (Integer) val; + if (max > v) { + min = v; + } else { + max = v; + } + } + + @Override + protected boolean isDisjoint(Range o) { + IntTestRange other = (IntTestRange)o; + return this.min > other.max || this.max < other.min; + } + + @Override + protected boolean isWithin(Range o) { + IntTestRange other = (IntTestRange)o; + if ((this.min >= other.min && this.max <= other.max) == false) { + // not within: + return false; + } + return true; + } + + @Override + protected boolean contains(Range o) { + IntTestRange other = (IntTestRange) o; + if ((this.min <= other.min && this.max >= other.max) == false) { + // not contains: + return false; + } + return true; + } + + } + +} diff --git a/core/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java b/core/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java new file mode 100644 index 00000000000..139cb3f0b12 --- /dev/null +++ b/core/src/test/java/org/apache/lucene/queries/LongRandomBinaryDocValuesRangeQueryTests.java @@ -0,0 +1,136 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lucene.queries; + +import org.apache.lucene.util.TestUtil; +import org.elasticsearch.index.mapper.RangeFieldMapper; + +public class LongRandomBinaryDocValuesRangeQueryTests extends BaseRandomBinaryDocValuesRangeQueryTestCase { + + @Override + protected String fieldName() { + return "long_range_dv_field"; + } + + @Override + protected RangeFieldMapper.RangeType rangeType() { + return RangeFieldMapper.RangeType.LONG; + } + + @Override + protected Range nextRange(int dimensions) throws Exception { + long value1 = nextLongInternal(); + long value2 = nextLongInternal(); + long min = Math.min(value1, value2); + long max = Math.max(value1, value2); + return new LongTestRange(min, max); + } + + private long nextLongInternal() { + switch (random().nextInt(5)) { + case 0: + return Long.MIN_VALUE; + case 1: + return Long.MAX_VALUE; + default: + int bpv = random().nextInt(64); + switch (bpv) { + case 64: + return random().nextLong(); + default: + long v = TestUtil.nextLong(random(), 0, (1L << bpv) - 1); + if (bpv > 0) { + // negative values sometimes + v -= 1L << (bpv - 1); + } + return v; + } + } + } + + private static class LongTestRange extends AbstractRange { + long min; + long max; + + LongTestRange(long min, long max) { + this.min = min; + this.max = max; + } + + @Override + public Object getMin() { + return min; + } + + @Override + protected void setMin(int dim, Object val) { + assert dim == 0; + long v = (Long)val; + if (min < v) { + max = v; + } else { + min = v; + } + } + + @Override + public Object getMax() { + return max; + } + + @Override + protected void setMax(int dim, Object val) { + assert dim == 0; + long v = (Long)val; + if (max > v) { + min = v; + } else { + max = v; + } + } + + @Override + protected boolean isDisjoint(Range o) { + LongTestRange other = (LongTestRange)o; + return this.min > other.max || this.max < other.min; + } + + @Override + protected boolean isWithin(Range o) { + LongTestRange other = (LongTestRange)o; + if ((this.min >= other.min && this.max <= other.max) == false) { + // not within: + return false; + } + return true; + } + + @Override + protected boolean contains(Range o) { + LongTestRange other = (LongTestRange) o; + if ((this.min <= other.min && this.max >= other.max) == false) { + // not contains: + return false; + } + return true; + } + + } + +} diff --git a/core/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java b/core/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java new file mode 100644 index 00000000000..8a4e6945ffc --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/mapper/BinaryRangeUtilTests.java @@ -0,0 +1,97 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.test.ESTestCase; + +public class BinaryRangeUtilTests extends ESTestCase { + + public void testBasics() { + BytesRef encoded1 = new BytesRef(BinaryRangeUtil.encode(Long.MIN_VALUE)); + BytesRef encoded2 = new BytesRef(BinaryRangeUtil.encode(-1L)); + BytesRef encoded3 = new BytesRef(BinaryRangeUtil.encode(0L)); + BytesRef encoded4 = new BytesRef(BinaryRangeUtil.encode(1L)); + BytesRef encoded5 = new BytesRef(BinaryRangeUtil.encode(Long.MAX_VALUE)); + + assertTrue(encoded1.compareTo(encoded2) < 0); + assertTrue(encoded2.compareTo(encoded1) > 0); + assertTrue(encoded2.compareTo(encoded3) < 0); + assertTrue(encoded3.compareTo(encoded2) > 0); + assertTrue(encoded3.compareTo(encoded4) < 0); + assertTrue(encoded4.compareTo(encoded3) > 0); + assertTrue(encoded4.compareTo(encoded5) < 0); + assertTrue(encoded5.compareTo(encoded4) > 0); + + encoded1 = new BytesRef(BinaryRangeUtil.encode(Double.NEGATIVE_INFINITY)); + encoded2 = new BytesRef(BinaryRangeUtil.encode(-1D)); + encoded3 = new BytesRef(BinaryRangeUtil.encode(0D)); + encoded4 = new BytesRef(BinaryRangeUtil.encode(1D)); + encoded5 = new BytesRef(BinaryRangeUtil.encode(Double.POSITIVE_INFINITY)); + + assertTrue(encoded1.compareTo(encoded2) < 0); + assertTrue(encoded2.compareTo(encoded1) > 0); + assertTrue(encoded2.compareTo(encoded3) < 0); + assertTrue(encoded3.compareTo(encoded2) > 0); + assertTrue(encoded3.compareTo(encoded4) < 0); + assertTrue(encoded4.compareTo(encoded3) > 0); + assertTrue(encoded4.compareTo(encoded5) < 0); + assertTrue(encoded5.compareTo(encoded4) > 0); + } + + public void testEncode_long() { + int iters = randomIntBetween(32, 1024); + for (int i = 0; i < iters; i++) { + long number1 = randomLong(); + BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encode(number1)); + long number2 = randomLong(); + BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encode(number2)); + + int cmp = normalize(Long.compare(number1, number2)); + assertEquals(cmp, normalize(encodedNumber1.compareTo(encodedNumber2))); + cmp = normalize(Long.compare(number2, number1)); + assertEquals(cmp, normalize(encodedNumber2.compareTo(encodedNumber1))); + } + } + + public void testEncode_double() { + int iters = randomIntBetween(32, 1024); + for (int i = 0; i < iters; i++) { + double number1 = randomDouble(); + BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encode(number1)); + double number2 = randomDouble(); + BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encode(number2)); + + int cmp = normalize(Double.compare(number1, number2)); + assertEquals(cmp, normalize(encodedNumber1.compareTo(encodedNumber2))); + cmp = normalize(Double.compare(number2, number1)); + assertEquals(cmp, normalize(encodedNumber2.compareTo(encodedNumber1))); + } + } + + private static int normalize(int cmp) { + if (cmp < 0) { + return -1; + } else if (cmp > 0) { + return 1; + } + return 0; + } + +} diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index a6fbfc44a56..9aa62d03703 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.network.InetAddresses; @@ -33,10 +34,10 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Locale; -import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; -import static org.elasticsearch.index.query.RangeQueryBuilder.LT_FIELD; +import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD; import static org.elasticsearch.index.query.RangeQueryBuilder.LTE_FIELD; +import static org.elasticsearch.index.query.RangeQueryBuilder.LT_FIELD; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; @@ -117,8 +118,11 @@ public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase { XContentType.JSON)); IndexableField[] fields = doc.rootDoc().getFields("field"); - assertEquals(1, fields.length); - IndexableField pointField = fields[0]; + assertEquals(2, fields.length); + IndexableField dvField = fields[0]; + assertEquals(DocValuesType.BINARY, dvField.fieldType().docValuesType()); + + IndexableField pointField = fields[1]; assertEquals(2, pointField.fieldType().pointDimensionCount()); assertFalse(pointField.fieldType().stored()); } @@ -145,7 +149,7 @@ public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase { XContentType.JSON)); IndexableField[] fields = doc.rootDoc().getFields("field"); - assertEquals(0, fields.length); + assertEquals(1, fields.length); } @Override @@ -195,10 +199,12 @@ public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase { XContentType.JSON)); IndexableField[] fields = doc.rootDoc().getFields("field"); - assertEquals(2, fields.length); - IndexableField pointField = fields[0]; + assertEquals(3, fields.length); + IndexableField dvField = fields[0]; + assertEquals(DocValuesType.BINARY, dvField.fieldType().docValuesType()); + IndexableField pointField = fields[1]; assertEquals(2, pointField.fieldType().pointDimensionCount()); - IndexableField storedField = fields[1]; + IndexableField storedField = fields[2]; assertTrue(storedField.fieldType().stored()); String strVal = "5"; if (type.equals("date_range")) { @@ -232,8 +238,10 @@ public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase { XContentType.JSON)); IndexableField[] fields = doc.rootDoc().getFields("field"); - assertEquals(1, fields.length); - IndexableField pointField = fields[0]; + assertEquals(2, fields.length); + IndexableField dvField = fields[0]; + assertEquals(DocValuesType.BINARY, dvField.fieldType().docValuesType()); + IndexableField pointField = fields[1]; assertEquals(2, pointField.fieldType().pointDimensionCount()); mapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -277,9 +285,9 @@ public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase { .endObject() .endObject().bytes(), XContentType.JSON)); - assertEquals(2, doc.rootDoc().getFields("field").length); + assertEquals(3, doc.rootDoc().getFields("field").length); IndexableField[] fields = doc.rootDoc().getFields("field"); - IndexableField storedField = fields[1]; + IndexableField storedField = fields[2]; String expected = type.equals("ip_range") ? InetAddresses.toAddrString((InetAddress)getMax(type)) : getMax(type) +""; assertThat(storedField.stringValue(), containsString(expected)); @@ -294,11 +302,13 @@ public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase { XContentType.JSON)); fields = doc.rootDoc().getFields("field"); - assertEquals(2, fields.length); - IndexableField pointField = fields[0]; + assertEquals(3, fields.length); + IndexableField dvField = fields[0]; + assertEquals(DocValuesType.BINARY, dvField.fieldType().docValuesType()); + IndexableField pointField = fields[1]; assertEquals(2, pointField.fieldType().pointDimensionCount()); assertFalse(pointField.fieldType().stored()); - storedField = fields[1]; + storedField = fields[2]; assertTrue(storedField.fieldType().stored()); String strVal = "5"; if (type.equals("date_range")) { @@ -336,11 +346,13 @@ public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase { XContentType.JSON)); IndexableField[] fields = doc.rootDoc().getFields("field"); - assertEquals(2, fields.length); - IndexableField pointField = fields[0]; + assertEquals(3, fields.length); + IndexableField dvField = fields[0]; + assertEquals(DocValuesType.BINARY, dvField.fieldType().docValuesType()); + IndexableField pointField = fields[1]; assertEquals(2, pointField.fieldType().pointDimensionCount()); assertFalse(pointField.fieldType().stored()); - IndexableField storedField = fields[1]; + IndexableField storedField = fields[2]; assertTrue(storedField.fieldType().stored()); String expected = type.equals("ip_range") ? InetAddresses.toAddrString((InetAddress)getMax(type)) : getMax(type) +""; assertThat(storedField.stringValue(), containsString(expected)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 015509d4a73..1a3606873c2 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -26,6 +26,8 @@ import org.apache.lucene.document.InetAddressRange; import org.apache.lucene.document.IntRange; import org.apache.lucene.document.LongRange; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.queries.BinaryDocValuesRangeQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.Query; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -70,7 +72,7 @@ public class RangeFieldTypeTests extends FieldTypeTestCase { @Override protected RangeFieldMapper.RangeFieldType createDefaultFieldType() { - return new RangeFieldMapper.RangeFieldType(type); + return new RangeFieldMapper.RangeFieldType(type, Version.CURRENT); } public void testRangeQuery() throws Exception { @@ -79,7 +81,7 @@ public class RangeFieldTypeTests extends FieldTypeTestCase { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); - RangeFieldMapper.RangeFieldType ft = new RangeFieldMapper.RangeFieldType(type); + RangeFieldMapper.RangeFieldType ft = new RangeFieldMapper.RangeFieldType(type, Version.CURRENT); ft.setName(FIELDNAME); ft.setIndexOptions(IndexOptions.DOCS); @@ -111,64 +113,125 @@ public class RangeFieldTypeTests extends FieldTypeTestCase { } private Query getDateRangeQuery(ShapeRelation relation, DateTime from, DateTime to, boolean includeLower, boolean includeUpper) { - return getLongRangeQuery(relation, from.getMillis(), to.getMillis(), includeLower, includeUpper); + long[] lower = new long[] {from.getMillis() + (includeLower ? 0 : 1)}; + long[] upper = new long[] {to.getMillis() - (includeUpper ? 0 : 1)}; + Query indexQuery; + BinaryDocValuesRangeQuery.QueryType queryType; + if (relation == ShapeRelation.WITHIN) { + indexQuery = LongRange.newWithinQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN; + } else if (relation == ShapeRelation.CONTAINS) { + indexQuery = LongRange.newContainsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS; + } else { + indexQuery = LongRange.newIntersectsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS; + } + Query dvQuery = RangeType.DATE.dvRangeQuery(FIELDNAME, queryType, from.getMillis(), + to.getMillis(), includeLower, includeUpper); + return new IndexOrDocValuesQuery(indexQuery, dvQuery); } private Query getIntRangeQuery(ShapeRelation relation, int from, int to, boolean includeLower, boolean includeUpper) { int[] lower = new int[] {from + (includeLower ? 0 : 1)}; int[] upper = new int[] {to - (includeUpper ? 0 : 1)}; + Query indexQuery; + BinaryDocValuesRangeQuery.QueryType queryType; if (relation == ShapeRelation.WITHIN) { - return IntRange.newWithinQuery(FIELDNAME, lower, upper); + indexQuery = IntRange.newWithinQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN; } else if (relation == ShapeRelation.CONTAINS) { - return IntRange.newContainsQuery(FIELDNAME, lower, upper); + indexQuery = IntRange.newContainsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS; + } else { + indexQuery = IntRange.newIntersectsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS; } - return IntRange.newIntersectsQuery(FIELDNAME, lower, upper); + Query dvQuery = RangeType.INTEGER.dvRangeQuery(FIELDNAME, queryType, from, to, + includeLower, includeUpper); + return new IndexOrDocValuesQuery(indexQuery, dvQuery); } private Query getLongRangeQuery(ShapeRelation relation, long from, long to, boolean includeLower, boolean includeUpper) { long[] lower = new long[] {from + (includeLower ? 0 : 1)}; long[] upper = new long[] {to - (includeUpper ? 0 : 1)}; + Query indexQuery; + BinaryDocValuesRangeQuery.QueryType queryType; if (relation == ShapeRelation.WITHIN) { - return LongRange.newWithinQuery(FIELDNAME, lower, upper); + indexQuery = LongRange.newWithinQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN; } else if (relation == ShapeRelation.CONTAINS) { - return LongRange.newContainsQuery(FIELDNAME, lower, upper); + indexQuery = LongRange.newContainsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS; + } else { + indexQuery = LongRange.newIntersectsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS; } - return LongRange.newIntersectsQuery(FIELDNAME, lower, upper); + Query dvQuery = RangeType.LONG.dvRangeQuery(FIELDNAME, queryType, from, to, + includeLower, includeUpper); + return new IndexOrDocValuesQuery(indexQuery, dvQuery); } private Query getFloatRangeQuery(ShapeRelation relation, float from, float to, boolean includeLower, boolean includeUpper) { float[] lower = new float[] {includeLower ? from : Math.nextUp(from)}; float[] upper = new float[] {includeUpper ? to : Math.nextDown(to)}; + Query indexQuery; + BinaryDocValuesRangeQuery.QueryType queryType; if (relation == ShapeRelation.WITHIN) { - return FloatRange.newWithinQuery(FIELDNAME, lower, upper); + indexQuery = FloatRange.newWithinQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN; } else if (relation == ShapeRelation.CONTAINS) { - return FloatRange.newContainsQuery(FIELDNAME, lower, upper); + indexQuery = FloatRange.newContainsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS; + } else { + indexQuery = FloatRange.newIntersectsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS; } - return FloatRange.newIntersectsQuery(FIELDNAME, lower, upper); + Query dvQuery = RangeType.FLOAT.dvRangeQuery(FIELDNAME, queryType, from, to, + includeLower, includeUpper); + return new IndexOrDocValuesQuery(indexQuery, dvQuery); } private Query getDoubleRangeQuery(ShapeRelation relation, double from, double to, boolean includeLower, boolean includeUpper) { double[] lower = new double[] {includeLower ? from : Math.nextUp(from)}; double[] upper = new double[] {includeUpper ? to : Math.nextDown(to)}; + Query indexQuery; + BinaryDocValuesRangeQuery.QueryType queryType; if (relation == ShapeRelation.WITHIN) { - return DoubleRange.newWithinQuery(FIELDNAME, lower, upper); + indexQuery = DoubleRange.newWithinQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN; } else if (relation == ShapeRelation.CONTAINS) { - return DoubleRange.newContainsQuery(FIELDNAME, lower, upper); + indexQuery = DoubleRange.newContainsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS; + } else { + indexQuery = DoubleRange.newIntersectsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS; } - return DoubleRange.newIntersectsQuery(FIELDNAME, lower, upper); + Query dvQuery = RangeType.DOUBLE.dvRangeQuery(FIELDNAME, queryType, from, to, + includeLower, includeUpper); + return new IndexOrDocValuesQuery(indexQuery, dvQuery); } private Query getInetAddressRangeQuery(ShapeRelation relation, InetAddress from, InetAddress to, boolean includeLower, boolean includeUpper) { InetAddress lower = includeLower ? from : InetAddressPoint.nextUp(from); InetAddress upper = includeUpper ? to : InetAddressPoint.nextDown(to); + Query indexQuery; + BinaryDocValuesRangeQuery.QueryType queryType; if (relation == ShapeRelation.WITHIN) { - return InetAddressRange.newWithinQuery(FIELDNAME, lower, upper); + indexQuery = InetAddressRange.newWithinQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.WITHIN; } else if (relation == ShapeRelation.CONTAINS) { - return InetAddressRange.newContainsQuery(FIELDNAME, lower, upper); + indexQuery = InetAddressRange.newContainsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.CONTAINS; + } else { + indexQuery = InetAddressRange.newIntersectsQuery(FIELDNAME, lower, upper); + queryType = BinaryDocValuesRangeQuery.QueryType.INTERSECTS; } - return InetAddressRange.newIntersectsQuery(FIELDNAME, lower, upper); + Query dvQuery = RangeType.IP.dvRangeQuery(FIELDNAME, queryType, from, to, + includeLower, includeUpper); + return new IndexOrDocValuesQuery(indexQuery, dvQuery); } private Object nextFrom() throws Exception {