More efficient encoding of range fields. (#26470)
This PR removes the vInt that precedes every value in order to know how long they are. Instead the query takes an enum that tells how to compute the length of values: for fixed-length data (ip addresses, double, float) the length is a constant while longs and integers use a variable-length representation that allows the length to be computed from the encoded values. Also the encoding of ints/longs was made a bit more efficient in order not to waste 3 bits in the header. As a consequence, values between -8 and 7 can now be encoded on 1 byte and values between -2048 and 2047 can now be encoded on 2 bytes or less. Closes #26443
This commit is contained in:
Adrien Grand
GitHub
parent
670849894f
commit
454cfc2cea
|
|
@ -37,15 +37,18 @@ public final class BinaryDocValuesRangeQuery extends Query {
|
|||
|
||||
private final String fieldName;
|
||||
private final QueryType queryType;
|
||||
private final LengthType lengthType;
|
||||
private final BytesRef from;
|
||||
private final BytesRef to;
|
||||
private final Object originalFrom;
|
||||
private final Object originalTo;
|
||||
|
||||
public BinaryDocValuesRangeQuery(String fieldName, QueryType queryType, BytesRef from, BytesRef to,
|
||||
public BinaryDocValuesRangeQuery(String fieldName, QueryType queryType, LengthType lengthType,
|
||||
BytesRef from, BytesRef to,
|
||||
Object originalFrom, Object originalTo) {
|
||||
this.fieldName = fieldName;
|
||||
this.queryType = queryType;
|
||||
this.lengthType = lengthType;
|
||||
this.from = from;
|
||||
this.to = to;
|
||||
this.originalFrom = originalFrom;
|
||||
|
|
@ -66,29 +69,34 @@ public final class BinaryDocValuesRangeQuery extends Query {
|
|||
final TwoPhaseIterator iterator = new TwoPhaseIterator(values) {
|
||||
|
||||
ByteArrayDataInput in = new ByteArrayDataInput();
|
||||
BytesRef otherFrom = new BytesRef(16);
|
||||
BytesRef otherTo = new BytesRef(16);
|
||||
BytesRef otherFrom = new BytesRef();
|
||||
BytesRef otherTo = new BytesRef();
|
||||
|
||||
@Override
|
||||
public boolean matches() throws IOException {
|
||||
BytesRef encodedRanges = values.binaryValue();
|
||||
in.reset(encodedRanges.bytes, encodedRanges.offset, encodedRanges.length);
|
||||
int numRanges = in.readVInt();
|
||||
final byte[] bytes = encodedRanges.bytes;
|
||||
otherFrom.bytes = bytes;
|
||||
otherTo.bytes = bytes;
|
||||
int offset = in.getPosition();
|
||||
for (int i = 0; i < numRanges; i++) {
|
||||
otherFrom.length = in.readVInt();
|
||||
otherFrom.bytes = encodedRanges.bytes;
|
||||
otherFrom.offset = in.getPosition();
|
||||
in.skipBytes(otherFrom.length);
|
||||
int length = lengthType.readLength(bytes, offset);
|
||||
otherFrom.offset = offset;
|
||||
otherFrom.length = length;
|
||||
offset += length;
|
||||
|
||||
otherTo.length = in.readVInt();
|
||||
otherTo.bytes = encodedRanges.bytes;
|
||||
otherTo.offset = in.getPosition();
|
||||
in.skipBytes(otherTo.length);
|
||||
length = lengthType.readLength(bytes, offset);
|
||||
otherTo.offset = offset;
|
||||
otherTo.length = length;
|
||||
offset += length;
|
||||
|
||||
if (queryType.matches(from, to, otherFrom, otherTo)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
assert offset == encodedRanges.offset + encodedRanges.length;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -114,13 +122,14 @@ public final class BinaryDocValuesRangeQuery extends Query {
|
|||
BinaryDocValuesRangeQuery that = (BinaryDocValuesRangeQuery) o;
|
||||
return Objects.equals(fieldName, that.fieldName) &&
|
||||
queryType == that.queryType &&
|
||||
lengthType == that.lengthType &&
|
||||
Objects.equals(from, that.from) &&
|
||||
Objects.equals(to, that.to);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(getClass(), fieldName, queryType, from, to);
|
||||
return Objects.hash(getClass(), fieldName, queryType, lengthType, from, to);
|
||||
}
|
||||
|
||||
public enum QueryType {
|
||||
|
|
@ -161,4 +170,42 @@ public final class BinaryDocValuesRangeQuery extends Query {
|
|||
|
||||
}
|
||||
|
||||
public enum LengthType {
|
||||
FIXED_4 {
|
||||
@Override
|
||||
int readLength(byte[] bytes, int offset) {
|
||||
return 4;
|
||||
}
|
||||
},
|
||||
FIXED_8 {
|
||||
@Override
|
||||
int readLength(byte[] bytes, int offset) {
|
||||
return 8;
|
||||
}
|
||||
},
|
||||
FIXED_16 {
|
||||
@Override
|
||||
int readLength(byte[] bytes, int offset) {
|
||||
return 16;
|
||||
}
|
||||
},
|
||||
VARIABLE {
|
||||
@Override
|
||||
int readLength(byte[] bytes, int offset) {
|
||||
// the first bit encodes the sign and the next 4 bits encode the number
|
||||
// of additional bytes
|
||||
int token = Byte.toUnsignedInt(bytes[offset]);
|
||||
int length = (token >>> 3) & 0x0f;
|
||||
if ((token & 0x80) == 0) {
|
||||
length = 0x0f - length;
|
||||
}
|
||||
return 1 + length;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Return the length of the value that starts at {@code offset} in {@code bytes}.
|
||||
*/
|
||||
abstract int readLength(byte[] bytes, int offset);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,11 +18,14 @@
|
|||
*/
|
||||
package org.elasticsearch.index.mapper;
|
||||
|
||||
import org.apache.lucene.document.HalfFloatPoint;
|
||||
import org.apache.lucene.store.ByteArrayDataOutput;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.NumericUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
|
|
@ -32,28 +35,17 @@ enum BinaryRangeUtil {
|
|||
|
||||
static BytesRef encodeLongRanges(Set<RangeFieldMapper.Range> ranges) throws IOException {
|
||||
List<RangeFieldMapper.Range> sortedRanges = new ArrayList<>(ranges);
|
||||
sortedRanges.sort((r1, r2) -> {
|
||||
long r1From = ((Number) r1.from).longValue();
|
||||
long r2From = ((Number) r2.from).longValue();
|
||||
int cmp = Long.compare(r1From, r2From);
|
||||
if (cmp != 0) {
|
||||
return cmp;
|
||||
} else {
|
||||
long r1To = ((Number) r1.from).longValue();
|
||||
long r2To = ((Number) r2.from).longValue();
|
||||
return Long.compare(r1To, r2To);
|
||||
}
|
||||
});
|
||||
Comparator<RangeFieldMapper.Range> fromComparator = Comparator.comparingLong(range -> ((Number) range.from).longValue());
|
||||
Comparator<RangeFieldMapper.Range> toComparator = Comparator.comparingLong(range -> ((Number) range.to).longValue());
|
||||
sortedRanges.sort(fromComparator.thenComparing(toComparator));
|
||||
|
||||
final byte[] encoded = new byte[5 + ((5 + 9) * 2) * sortedRanges.size()];
|
||||
final byte[] encoded = new byte[5 + (9 * 2) * sortedRanges.size()];
|
||||
ByteArrayDataOutput out = new ByteArrayDataOutput(encoded);
|
||||
out.writeVInt(sortedRanges.size());
|
||||
for (RangeFieldMapper.Range range : sortedRanges) {
|
||||
byte[] encodedFrom = encode(((Number) range.from).longValue());
|
||||
out.writeVInt(encodedFrom.length);
|
||||
byte[] encodedFrom = encodeLong(((Number) range.from).longValue());
|
||||
out.writeBytes(encodedFrom, encodedFrom.length);
|
||||
byte[] encodedTo = encode(((Number) range.to).longValue());
|
||||
out.writeVInt(encodedTo.length);
|
||||
byte[] encodedTo = encodeLong(((Number) range.to).longValue());
|
||||
out.writeBytes(encodedTo, encodedTo.length);
|
||||
}
|
||||
return new BytesRef(encoded, 0, out.getPosition());
|
||||
|
|
@ -61,38 +53,59 @@ enum BinaryRangeUtil {
|
|||
|
||||
static BytesRef encodeDoubleRanges(Set<RangeFieldMapper.Range> ranges) throws IOException {
|
||||
List<RangeFieldMapper.Range> sortedRanges = new ArrayList<>(ranges);
|
||||
sortedRanges.sort((r1, r2) -> {
|
||||
double r1From = ((Number) r1.from).doubleValue();
|
||||
double r2From = ((Number) r2.from).doubleValue();
|
||||
int cmp = Double.compare(r1From, r2From);
|
||||
if (cmp != 0) {
|
||||
return cmp;
|
||||
} else {
|
||||
double r1To = ((Number) r1.from).doubleValue();
|
||||
double r2To = ((Number) r2.from).doubleValue();
|
||||
return Double.compare(r1To, r2To);
|
||||
}
|
||||
});
|
||||
Comparator<RangeFieldMapper.Range> fromComparator = Comparator.comparingDouble(range -> ((Number) range.from).doubleValue());
|
||||
Comparator<RangeFieldMapper.Range> toComparator = Comparator.comparingDouble(range -> ((Number) range.to).doubleValue());
|
||||
sortedRanges.sort(fromComparator.thenComparing(toComparator));
|
||||
|
||||
final byte[] encoded = new byte[5 + ((5 + 9) * 2) * sortedRanges.size()];
|
||||
final byte[] encoded = new byte[5 + (8 * 2) * sortedRanges.size()];
|
||||
ByteArrayDataOutput out = new ByteArrayDataOutput(encoded);
|
||||
out.writeVInt(sortedRanges.size());
|
||||
for (RangeFieldMapper.Range range : sortedRanges) {
|
||||
byte[] encodedFrom = BinaryRangeUtil.encode(((Number) range.from).doubleValue());
|
||||
out.writeVInt(encodedFrom.length);
|
||||
byte[] encodedFrom = encodeDouble(((Number) range.from).doubleValue());
|
||||
out.writeBytes(encodedFrom, encodedFrom.length);
|
||||
byte[] encodedTo = BinaryRangeUtil.encode(((Number) range.to).doubleValue());
|
||||
out.writeVInt(encodedTo.length);
|
||||
byte[] encodedTo = encodeDouble(((Number) range.to).doubleValue());
|
||||
out.writeBytes(encodedTo, encodedTo.length);
|
||||
}
|
||||
return new BytesRef(encoded, 0, out.getPosition());
|
||||
}
|
||||
|
||||
static BytesRef encodeFloatRanges(Set<RangeFieldMapper.Range> ranges) throws IOException {
|
||||
List<RangeFieldMapper.Range> sortedRanges = new ArrayList<>(ranges);
|
||||
Comparator<RangeFieldMapper.Range> fromComparator = Comparator.comparingDouble(range -> ((Number) range.from).floatValue());
|
||||
Comparator<RangeFieldMapper.Range> toComparator = Comparator.comparingDouble(range -> ((Number) range.to).floatValue());
|
||||
sortedRanges.sort(fromComparator.thenComparing(toComparator));
|
||||
|
||||
final byte[] encoded = new byte[5 + (4 * 2) * sortedRanges.size()];
|
||||
ByteArrayDataOutput out = new ByteArrayDataOutput(encoded);
|
||||
out.writeVInt(sortedRanges.size());
|
||||
for (RangeFieldMapper.Range range : sortedRanges) {
|
||||
byte[] encodedFrom = encodeFloat(((Number) range.from).floatValue());
|
||||
out.writeBytes(encodedFrom, encodedFrom.length);
|
||||
byte[] encodedTo = encodeFloat(((Number) range.to).floatValue());
|
||||
out.writeBytes(encodedTo, encodedTo.length);
|
||||
}
|
||||
return new BytesRef(encoded, 0, out.getPosition());
|
||||
}
|
||||
|
||||
static byte[] encodeDouble(double number) {
|
||||
byte[] encoded = new byte[8];
|
||||
NumericUtils.longToSortableBytes(NumericUtils.doubleToSortableLong(number), encoded, 0);
|
||||
return encoded;
|
||||
}
|
||||
|
||||
static byte[] encodeFloat(float number) {
|
||||
byte[] encoded = new byte[4];
|
||||
NumericUtils.intToSortableBytes(NumericUtils.floatToSortableInt(number), encoded, 0);
|
||||
return encoded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes the specified number of type long in a variable-length byte format.
|
||||
* The byte format preserves ordering, which means the returned byte array can be used for comparing as is.
|
||||
* The first bit stores the sign and the 4 subsequent bits encode the number of bytes that are used to
|
||||
* represent the long value, in addition to the first one.
|
||||
*/
|
||||
static byte[] encode(long number) {
|
||||
static byte[] encodeLong(long number) {
|
||||
int sign = 1; // means positive
|
||||
if (number < 0) {
|
||||
number = -1 - number;
|
||||
|
|
@ -101,46 +114,48 @@ enum BinaryRangeUtil {
|
|||
return encode(number, sign);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes the specified number of type double in a variable-length byte format.
|
||||
* The byte format preserves ordering, which means the returned byte array can be used for comparing as is.
|
||||
*/
|
||||
static byte[] encode(double number) {
|
||||
long l;
|
||||
int sign;
|
||||
if (number < 0.0) {
|
||||
l = Double.doubleToRawLongBits(-0d - number);
|
||||
sign = 0;
|
||||
} else {
|
||||
l = Double.doubleToRawLongBits(number);
|
||||
sign = 1; // means positive
|
||||
}
|
||||
return encode(l, sign);
|
||||
}
|
||||
|
||||
private static byte[] encode(long l, int sign) {
|
||||
assert l >= 0;
|
||||
int bits = 64 - Long.numberOfLeadingZeros(l);
|
||||
|
||||
int numBytes = (bits + 7) / 8; // between 0 and 8
|
||||
byte[] encoded = new byte[1 + numBytes];
|
||||
// encode the sign first to make sure positive values compare greater than negative values
|
||||
// and then the number of bytes, to make sure that large values compare greater than low values
|
||||
if (sign > 0) {
|
||||
encoded[0] = (byte) ((sign << 4) | numBytes);
|
||||
} else {
|
||||
encoded[0] = (byte) ((sign << 4) | (8 - numBytes));
|
||||
// the header is formed of:
|
||||
// - 1 bit for the sign
|
||||
// - 4 bits for the number of additional bytes
|
||||
// - up to 3 bits of the value
|
||||
// additional bytes are data bytes
|
||||
|
||||
int numBits = 64 - Long.numberOfLeadingZeros(l);
|
||||
int numAdditionalBytes = (numBits + 7 - 3) / 8;
|
||||
|
||||
byte[] encoded = new byte[1 + numAdditionalBytes];
|
||||
|
||||
// write data bytes
|
||||
int i = encoded.length;
|
||||
while (numBits > 0) {
|
||||
int index = --i;
|
||||
assert index > 0 || numBits <= 3; // byte 0 can't encode more than 3 bits
|
||||
encoded[index] = (byte) l;
|
||||
l >>>= 8;
|
||||
numBits -= 8;
|
||||
}
|
||||
for (int b = 0; b < numBytes; ++b) {
|
||||
if (sign == 1) {
|
||||
encoded[encoded.length - 1 - b] = (byte) (l >>> (8 * b));
|
||||
} else if (sign == 0) {
|
||||
encoded[encoded.length - 1 - b] = (byte) (0xFF - ((l >>> (8 * b)) & 0xFF));
|
||||
} else {
|
||||
throw new AssertionError();
|
||||
assert Byte.toUnsignedInt(encoded[0]) <= 0x07;
|
||||
assert encoded.length == 1 || encoded[0] != 0 || Byte.toUnsignedInt(encoded[1]) > 0x07;
|
||||
|
||||
if (sign == 0) {
|
||||
// reverse the order
|
||||
for (int j = 0; j < encoded.length; ++j) {
|
||||
encoded[j] = (byte) ~Byte.toUnsignedInt(encoded[j]);
|
||||
}
|
||||
// the first byte only uses 3 bits, we need the 5 upper bits for the header
|
||||
encoded[0] &= 0x07;
|
||||
}
|
||||
|
||||
// write the header
|
||||
encoded[0] |= sign << 7;
|
||||
if (sign > 0) {
|
||||
encoded[0] |= numAdditionalBytes << 3;
|
||||
} else {
|
||||
encoded[0] |= (15 - numAdditionalBytes) << 3;
|
||||
}
|
||||
return encoded;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -477,12 +477,10 @@ public class RangeFieldMapper extends FieldMapper {
|
|||
ByteArrayDataOutput out = new ByteArrayDataOutput(encoded);
|
||||
out.writeVInt(ranges.size());
|
||||
for (Range range : ranges) {
|
||||
out.writeVInt(16);
|
||||
InetAddress fromValue = (InetAddress) range.from;
|
||||
byte[] encodedFromValue = InetAddressPoint.encode(fromValue);
|
||||
out.writeBytes(encodedFromValue, 0, encodedFromValue.length);
|
||||
|
||||
out.writeVInt(16);
|
||||
InetAddress toValue = (InetAddress) range.to;
|
||||
byte[] encodedToValue = InetAddressPoint.encode(toValue);
|
||||
out.writeBytes(encodedToValue, 0, encodedToValue.length);
|
||||
|
|
@ -491,10 +489,19 @@ public class RangeFieldMapper extends FieldMapper {
|
|||
}
|
||||
|
||||
@Override
|
||||
BytesRef[] encodeRange(Object from, Object to) {
|
||||
BytesRef encodedFrom = new BytesRef(InetAddressPoint.encode((InetAddress) from));
|
||||
BytesRef encodedTo = new BytesRef(InetAddressPoint.encode((InetAddress) to));
|
||||
return new BytesRef[]{encodedFrom, encodedTo};
|
||||
public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) {
|
||||
if (includeFrom == false) {
|
||||
from = nextUp(from);
|
||||
}
|
||||
|
||||
if (includeTo == false) {
|
||||
to = nextDown(to);
|
||||
}
|
||||
|
||||
byte[] encodedFrom = InetAddressPoint.encode((InetAddress) from);
|
||||
byte[] encodedTo = InetAddressPoint.encode((InetAddress) to);
|
||||
return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.FIXED_16,
|
||||
new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -565,8 +572,8 @@ public class RangeFieldMapper extends FieldMapper {
|
|||
}
|
||||
|
||||
@Override
|
||||
BytesRef[] encodeRange(Object from, Object to) {
|
||||
return LONG.encodeRange(from, to);
|
||||
public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) {
|
||||
return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -620,12 +627,23 @@ public class RangeFieldMapper extends FieldMapper {
|
|||
|
||||
@Override
|
||||
public BytesRef encodeRanges(Set<Range> ranges) throws IOException {
|
||||
return DOUBLE.encodeRanges(ranges);
|
||||
return BinaryRangeUtil.encodeFloatRanges(ranges);
|
||||
}
|
||||
|
||||
@Override
|
||||
BytesRef[] encodeRange(Object from, Object to) {
|
||||
return DOUBLE.encodeRange(((Number) from).floatValue(), ((Number) to).floatValue());
|
||||
public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) {
|
||||
if (includeFrom == false) {
|
||||
from = nextUp(from);
|
||||
}
|
||||
|
||||
if (includeTo == false) {
|
||||
to = nextDown(to);
|
||||
}
|
||||
|
||||
byte[] encodedFrom = BinaryRangeUtil.encodeFloat((Float) from);
|
||||
byte[] encodedTo = BinaryRangeUtil.encodeFloat((Float) to);
|
||||
return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.FIXED_4,
|
||||
new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -675,10 +693,19 @@ public class RangeFieldMapper extends FieldMapper {
|
|||
}
|
||||
|
||||
@Override
|
||||
BytesRef[] encodeRange(Object from, Object to) {
|
||||
byte[] fromValue = BinaryRangeUtil.encode(((Number) from).doubleValue());
|
||||
byte[] toValue = BinaryRangeUtil.encode(((Number) to).doubleValue());
|
||||
return new BytesRef[]{new BytesRef(fromValue), new BytesRef(toValue)};
|
||||
public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) {
|
||||
if (includeFrom == false) {
|
||||
from = nextUp(from);
|
||||
}
|
||||
|
||||
if (includeTo == false) {
|
||||
to = nextDown(to);
|
||||
}
|
||||
|
||||
byte[] encodedFrom = BinaryRangeUtil.encodeDouble((Double) from);
|
||||
byte[] encodedTo = BinaryRangeUtil.encodeDouble((Double) to);
|
||||
return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.FIXED_8,
|
||||
new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -730,8 +757,8 @@ public class RangeFieldMapper extends FieldMapper {
|
|||
}
|
||||
|
||||
@Override
|
||||
BytesRef[] encodeRange(Object from, Object to) {
|
||||
return LONG.encodeRange(from, to);
|
||||
public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) {
|
||||
return LONG.dvRangeQuery(field, queryType, from, to, includeFrom, includeTo);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -778,10 +805,19 @@ public class RangeFieldMapper extends FieldMapper {
|
|||
}
|
||||
|
||||
@Override
|
||||
BytesRef[] encodeRange(Object from, Object to) {
|
||||
byte[] encodedFrom = BinaryRangeUtil.encode(((Number) from).longValue());
|
||||
byte[] encodedTo = BinaryRangeUtil.encode(((Number) to).longValue());
|
||||
return new BytesRef[]{new BytesRef(encodedFrom), new BytesRef(encodedTo)};
|
||||
public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) {
|
||||
if (includeFrom == false) {
|
||||
from = nextUp(from);
|
||||
}
|
||||
|
||||
if (includeTo == false) {
|
||||
to = nextDown(to);
|
||||
}
|
||||
|
||||
byte[] encodedFrom = BinaryRangeUtil.encodeLong(((Number) from).longValue());
|
||||
byte[] encodedTo = BinaryRangeUtil.encodeLong(((Number) to).longValue());
|
||||
return new BinaryDocValuesRangeQuery(field, queryType, BinaryDocValuesRangeQuery.LengthType.VARIABLE,
|
||||
new BytesRef(encodedFrom), new BytesRef(encodedTo), from, to);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -897,19 +933,8 @@ public class RangeFieldMapper extends FieldMapper {
|
|||
// rounded up via parseFrom and parseTo methods.
|
||||
public abstract BytesRef encodeRanges(Set<Range> ranges) throws IOException;
|
||||
|
||||
public Query dvRangeQuery(String field, QueryType queryType, Object from, Object to, boolean includeFrom, boolean includeTo) {
|
||||
if (includeFrom == false) {
|
||||
from = nextUp(from);
|
||||
}
|
||||
|
||||
if (includeTo == false) {
|
||||
to = nextDown(to);
|
||||
}
|
||||
BytesRef[] range = encodeRange(from, to);
|
||||
return new BinaryDocValuesRangeQuery(field, queryType, range[0], range[1], from, to);
|
||||
}
|
||||
|
||||
abstract BytesRef[] encodeRange(Object from, Object to);
|
||||
public abstract Query dvRangeQuery(String field, QueryType queryType, Object from, Object to,
|
||||
boolean includeFrom, boolean includeTo);
|
||||
|
||||
public final String name;
|
||||
private final NumberType numberType;
|
||||
|
|
|
|||
|
|
@ -24,11 +24,11 @@ import org.elasticsearch.test.ESTestCase;
|
|||
public class BinaryRangeUtilTests extends ESTestCase {
|
||||
|
||||
public void testBasics() {
|
||||
BytesRef encoded1 = new BytesRef(BinaryRangeUtil.encode(Long.MIN_VALUE));
|
||||
BytesRef encoded2 = new BytesRef(BinaryRangeUtil.encode(-1L));
|
||||
BytesRef encoded3 = new BytesRef(BinaryRangeUtil.encode(0L));
|
||||
BytesRef encoded4 = new BytesRef(BinaryRangeUtil.encode(1L));
|
||||
BytesRef encoded5 = new BytesRef(BinaryRangeUtil.encode(Long.MAX_VALUE));
|
||||
BytesRef encoded1 = new BytesRef(BinaryRangeUtil.encodeLong(Long.MIN_VALUE));
|
||||
BytesRef encoded2 = new BytesRef(BinaryRangeUtil.encodeLong(-1L));
|
||||
BytesRef encoded3 = new BytesRef(BinaryRangeUtil.encodeLong(0L));
|
||||
BytesRef encoded4 = new BytesRef(BinaryRangeUtil.encodeLong(1L));
|
||||
BytesRef encoded5 = new BytesRef(BinaryRangeUtil.encodeLong(Long.MAX_VALUE));
|
||||
|
||||
assertTrue(encoded1.compareTo(encoded2) < 0);
|
||||
assertTrue(encoded2.compareTo(encoded1) > 0);
|
||||
|
|
@ -39,11 +39,12 @@ public class BinaryRangeUtilTests extends ESTestCase {
|
|||
assertTrue(encoded4.compareTo(encoded5) < 0);
|
||||
assertTrue(encoded5.compareTo(encoded4) > 0);
|
||||
|
||||
encoded1 = new BytesRef(BinaryRangeUtil.encode(Double.NEGATIVE_INFINITY));
|
||||
encoded2 = new BytesRef(BinaryRangeUtil.encode(-1D));
|
||||
encoded3 = new BytesRef(BinaryRangeUtil.encode(0D));
|
||||
encoded4 = new BytesRef(BinaryRangeUtil.encode(1D));
|
||||
encoded5 = new BytesRef(BinaryRangeUtil.encode(Double.POSITIVE_INFINITY));
|
||||
encoded1 = new BytesRef(BinaryRangeUtil.encodeDouble(Double.NEGATIVE_INFINITY));
|
||||
encoded2 = new BytesRef(BinaryRangeUtil.encodeDouble(-1D));
|
||||
encoded3 = new BytesRef(BinaryRangeUtil.encodeDouble(-0D));
|
||||
encoded4 = new BytesRef(BinaryRangeUtil.encodeDouble(0D));
|
||||
encoded5 = new BytesRef(BinaryRangeUtil.encodeDouble(1D));
|
||||
BytesRef encoded6 = new BytesRef(BinaryRangeUtil.encodeDouble(Double.POSITIVE_INFINITY));
|
||||
|
||||
assertTrue(encoded1.compareTo(encoded2) < 0);
|
||||
assertTrue(encoded2.compareTo(encoded1) > 0);
|
||||
|
|
@ -53,15 +54,35 @@ public class BinaryRangeUtilTests extends ESTestCase {
|
|||
assertTrue(encoded4.compareTo(encoded3) > 0);
|
||||
assertTrue(encoded4.compareTo(encoded5) < 0);
|
||||
assertTrue(encoded5.compareTo(encoded4) > 0);
|
||||
assertTrue(encoded5.compareTo(encoded6) < 0);
|
||||
assertTrue(encoded6.compareTo(encoded5) > 0);
|
||||
|
||||
encoded1 = new BytesRef(BinaryRangeUtil.encodeFloat(Float.NEGATIVE_INFINITY));
|
||||
encoded2 = new BytesRef(BinaryRangeUtil.encodeFloat(-1F));
|
||||
encoded3 = new BytesRef(BinaryRangeUtil.encodeFloat(-0F));
|
||||
encoded4 = new BytesRef(BinaryRangeUtil.encodeFloat(0F));
|
||||
encoded5 = new BytesRef(BinaryRangeUtil.encodeFloat(1F));
|
||||
encoded6 = new BytesRef(BinaryRangeUtil.encodeFloat(Float.POSITIVE_INFINITY));
|
||||
|
||||
assertTrue(encoded1.compareTo(encoded2) < 0);
|
||||
assertTrue(encoded2.compareTo(encoded1) > 0);
|
||||
assertTrue(encoded2.compareTo(encoded3) < 0);
|
||||
assertTrue(encoded3.compareTo(encoded2) > 0);
|
||||
assertTrue(encoded3.compareTo(encoded4) < 0);
|
||||
assertTrue(encoded4.compareTo(encoded3) > 0);
|
||||
assertTrue(encoded4.compareTo(encoded5) < 0);
|
||||
assertTrue(encoded5.compareTo(encoded4) > 0);
|
||||
assertTrue(encoded5.compareTo(encoded6) < 0);
|
||||
assertTrue(encoded6.compareTo(encoded5) > 0);
|
||||
}
|
||||
|
||||
public void testEncode_long() {
|
||||
int iters = randomIntBetween(32, 1024);
|
||||
for (int i = 0; i < iters; i++) {
|
||||
long number1 = randomLong();
|
||||
BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encode(number1));
|
||||
long number2 = randomLong();
|
||||
BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encode(number2));
|
||||
BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encodeLong(number1));
|
||||
long number2 = randomBoolean() ? number1 + 1 : randomLong();
|
||||
BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encodeLong(number2));
|
||||
|
||||
int cmp = normalize(Long.compare(number1, number2));
|
||||
assertEquals(cmp, normalize(encodedNumber1.compareTo(encodedNumber2)));
|
||||
|
|
@ -70,14 +91,48 @@ public class BinaryRangeUtilTests extends ESTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
public void testVariableLengthEncoding() {
|
||||
for (int i = -8; i <= 7; ++i) {
|
||||
assertEquals(1, BinaryRangeUtil.encodeLong(i).length);
|
||||
}
|
||||
for (int i = -2048; i <= 2047; ++i) {
|
||||
if (i < -8 ||i > 7) {
|
||||
assertEquals(2, BinaryRangeUtil.encodeLong(i).length);
|
||||
}
|
||||
}
|
||||
assertEquals(3, BinaryRangeUtil.encodeLong(-2049).length);
|
||||
assertEquals(3, BinaryRangeUtil.encodeLong(2048).length);
|
||||
assertEquals(9, BinaryRangeUtil.encodeLong(Long.MIN_VALUE).length);
|
||||
assertEquals(9, BinaryRangeUtil.encodeLong(Long.MAX_VALUE).length);
|
||||
}
|
||||
|
||||
public void testEncode_double() {
|
||||
int iters = randomIntBetween(32, 1024);
|
||||
for (int i = 0; i < iters; i++) {
|
||||
double number1 = randomDouble();
|
||||
BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encode(number1));
|
||||
double number2 = randomDouble();
|
||||
BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encode(number2));
|
||||
BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encodeDouble(number1));
|
||||
double number2 = randomBoolean() ? Math.nextUp(number1) : randomDouble();
|
||||
BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encodeDouble(number2));
|
||||
|
||||
assertEquals(8, encodedNumber1.length);
|
||||
assertEquals(8, encodedNumber2.length);
|
||||
int cmp = normalize(Double.compare(number1, number2));
|
||||
assertEquals(cmp, normalize(encodedNumber1.compareTo(encodedNumber2)));
|
||||
cmp = normalize(Double.compare(number2, number1));
|
||||
assertEquals(cmp, normalize(encodedNumber2.compareTo(encodedNumber1)));
|
||||
}
|
||||
}
|
||||
|
||||
public void testEncode_Float() {
|
||||
int iters = randomIntBetween(32, 1024);
|
||||
for (int i = 0; i < iters; i++) {
|
||||
float number1 = randomFloat();
|
||||
BytesRef encodedNumber1 = new BytesRef(BinaryRangeUtil.encodeFloat(number1));
|
||||
float number2 = randomBoolean() ? Math.nextUp(number1) : randomFloat();
|
||||
BytesRef encodedNumber2 = new BytesRef(BinaryRangeUtil.encodeFloat(number2));
|
||||
|
||||
assertEquals(4, encodedNumber1.length);
|
||||
assertEquals(4, encodedNumber2.length);
|
||||
int cmp = normalize(Double.compare(number1, number2));
|
||||
assertEquals(cmp, normalize(encodedNumber1.compareTo(encodedNumber2)));
|
||||
cmp = normalize(Double.compare(number2, number1));
|
||||
|
|
|
|||
Loading…
Reference in New Issue