honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-17 10:25:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

SQL: improve conversion of Date types (elastic/x-pack-elasticsearch#4382)

Browse Source 
When dealing with dates, the conversion now returns a proper DateTime
instance instead of a long

Relates elastic/x-pack-elasticsearch#4331

Original commit: elastic/x-pack-elasticsearch@bba9f2c79f
This commit is contained in:
Costin Leau 2018-04-16 19:58:32 +03:00 committed by GitHub
parent 25895e0a3c
commit f7bed219f3
3 changed files with 118 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFunction.java
View File

@ -19,6 +19,7 @@ import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTempl
import org.elasticsearch.xpack.sql.tree.Location;
import org.elasticsearch.xpack.sql.tree.NodeInfo;
import org.elasticsearch.xpack.sql.type.DataType;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.time.Instant;
@ -63,12 +64,13 @@ public abstract class DateTimeFunction extends UnaryScalarFunction {
@Override
public Object fold() {
Long folded = (Long) field().fold();
DateTime folded = (DateTime) field().fold();
if (folded == null) {
return null;
}
ZonedDateTime time = ZonedDateTime.ofInstant(
Instant.ofEpochMilli(folded), ZoneId.of(timeZone.getID()));
Instant.ofEpochMilli(folded.getMillis()), ZoneId.of(timeZone.getID()));
return time.get(chronoField());
}

93
plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java
View File

@ -7,6 +7,9 @@ package org.elasticsearch.xpack.sql.type;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.ReadableInstant;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
@ -15,6 +18,10 @@ import java.util.function.DoubleFunction;
import java.util.function.Function;
import java.util.function.LongFunction;
import static org.elasticsearch.xpack.sql.type.DataType.BOOLEAN;
import static org.elasticsearch.xpack.sql.type.DataType.DATE;
import static org.elasticsearch.xpack.sql.type.DataType.LONG;
import static org.elasticsearch.xpack.sql.type.DataType.NULL;
/**
* Conversions from one Elasticsearch data type to another Elasticsearch data types.
* <p>
@ -78,7 +85,7 @@ public abstract class DataTypeConversion {
*/
public static boolean canConvert(DataType from, DataType to) {
// Special handling for nulls and if conversion is not requires
if (from == to || from == DataType.NULL) {
if (from == to || from == NULL) {
return true;
}
// only primitives are supported so far
@ -96,7 +103,7 @@ public abstract class DataTypeConversion {
if (to == DataType.NULL) {
return Conversion.NULL;
}
Conversion conversion = conversion(from, to);
if (conversion == null) {
throw new SqlIllegalArgumentException("cannot convert from [" + from + "] to [" + to + "]");
@ -132,7 +139,7 @@ public abstract class DataTypeConversion {
}
private static Conversion conversionToString(DataType from) {
if (from == DataType.DATE) {
if (from == DATE) {
return Conversion.DATE_TO_STRING;
}
return Conversion.OTHER_TO_STRING;
@ -145,12 +152,15 @@ public abstract class DataTypeConversion {
if (from.isInteger) {
return Conversion.INTEGER_TO_LONG;
}
if (from == DataType.BOOLEAN) {
if (from == BOOLEAN) {
return Conversion.BOOL_TO_INT; // We emit an int here which is ok because of Java's casting rules
}
if (from.isString()) {
return Conversion.STRING_TO_LONG;
}
if (from == DATE) {
return Conversion.DATE_TO_LONG;
}
return null;
}
@ -161,12 +171,15 @@ public abstract class DataTypeConversion {
if (from.isInteger) {
return Conversion.INTEGER_TO_INT;
}
if (from == DataType.BOOLEAN) {
if (from == BOOLEAN) {
return Conversion.BOOL_TO_INT;
}
if (from.isString()) {
return Conversion.STRING_TO_INT;
}
if (from == DATE) {
return Conversion.DATE_TO_INT;
}
return null;
}
@ -177,12 +190,15 @@ public abstract class DataTypeConversion {
if (from.isInteger) {
return Conversion.INTEGER_TO_SHORT;
}
if (from == DataType.BOOLEAN) {
if (from == BOOLEAN) {
return Conversion.BOOL_TO_SHORT;
}
if (from.isString()) {
return Conversion.STRING_TO_SHORT;
}
if (from == DATE) {
return Conversion.DATE_TO_SHORT;
}
return null;
}
@ -193,12 +209,15 @@ public abstract class DataTypeConversion {
if (from.isInteger) {
return Conversion.INTEGER_TO_BYTE;
}
if (from == DataType.BOOLEAN) {
if (from == BOOLEAN) {
return Conversion.BOOL_TO_BYTE;
}
if (from.isString()) {
return Conversion.STRING_TO_BYTE;
}
if (from == DATE) {
return Conversion.DATE_TO_BYTE;
}
return null;
}
@ -209,12 +228,15 @@ public abstract class DataTypeConversion {
if (from.isInteger) {
return Conversion.INTEGER_TO_FLOAT;
}
if (from == DataType.BOOLEAN) {
if (from == BOOLEAN) {
return Conversion.BOOL_TO_FLOAT;
}
if (from.isString()) {
return Conversion.STRING_TO_FLOAT;
}
if (from == DATE) {
return Conversion.DATE_TO_FLOAT;
}
return null;
}
@ -225,24 +247,27 @@ public abstract class DataTypeConversion {
if (from.isInteger) {
return Conversion.INTEGER_TO_DOUBLE;
}
if (from == DataType.BOOLEAN) {
if (from == BOOLEAN) {
return Conversion.BOOL_TO_DOUBLE;
}
if (from.isString()) {
return Conversion.STRING_TO_DOUBLE;
}
if (from == DATE) {
return Conversion.DATE_TO_DOUBLE;
}
return null;
}
private static Conversion conversionToDate(DataType from) {
if (from.isRational) {
return Conversion.RATIONAL_TO_LONG;
return Conversion.RATIONAL_TO_DATE;
}
if (from.isInteger) {
return Conversion.INTEGER_TO_LONG;
return Conversion.INTEGER_TO_DATE;
}
if (from == DataType.BOOLEAN) {
return Conversion.BOOL_TO_INT; // We emit an int here which is ok because of Java's casting rules
if (from == BOOLEAN) {
return Conversion.BOOL_TO_DATE; // We emit an int here which is ok because of Java's casting rules
}
if (from.isString()) {
return Conversion.STRING_TO_DATE;
@ -257,6 +282,9 @@ public abstract class DataTypeConversion {
if (from.isString()) {
return Conversion.STRING_TO_BOOLEAN;
}
if (from == DATE) {
return Conversion.DATE_TO_BOOLEAN;
}
return null;
}
@ -316,35 +344,54 @@ public abstract class DataTypeConversion {
public enum Conversion {
IDENTITY(Function.identity()),
NULL(value -> null),
DATE_TO_STRING(Object::toString),
OTHER_TO_STRING(String::valueOf),
RATIONAL_TO_LONG(fromDouble(DataTypeConversion::safeToLong)),
INTEGER_TO_LONG(fromLong(value -> value)),
STRING_TO_LONG(fromString(Long::valueOf, "Long")),
DATE_TO_LONG(fromDate(value -> value)),
RATIONAL_TO_INT(fromDouble(value -> safeToInt(safeToLong(value)))),
INTEGER_TO_INT(fromLong(DataTypeConversion::safeToInt)),
BOOL_TO_INT(fromBool(value -> value ? 1 : 0)),
STRING_TO_INT(fromString(Integer::valueOf, "Int")),
DATE_TO_INT(fromDate(DataTypeConversion::safeToInt)),
RATIONAL_TO_SHORT(fromDouble(value -> safeToShort(safeToLong(value)))),
INTEGER_TO_SHORT(fromLong(DataTypeConversion::safeToShort)),
BOOL_TO_SHORT(fromBool(value -> value ? (short) 1 : (short) 0)),
STRING_TO_SHORT(fromString(Short::valueOf, "Short")),
DATE_TO_SHORT(fromDate(DataTypeConversion::safeToShort)),
RATIONAL_TO_BYTE(fromDouble(value -> safeToByte(safeToLong(value)))),
INTEGER_TO_BYTE(fromLong(DataTypeConversion::safeToByte)),
BOOL_TO_BYTE(fromBool(value -> value ? (byte) 1 : (byte) 0)),
STRING_TO_BYTE(fromString(Byte::valueOf, "Byte")),
DATE_TO_BYTE(fromDate(DataTypeConversion::safeToByte)),
// TODO floating point conversions are lossy but conversions to integer conversions are not. Are we ok with that?
RATIONAL_TO_FLOAT(fromDouble(value -> (float) value)),
INTEGER_TO_FLOAT(fromLong(value -> (float) value)),
BOOL_TO_FLOAT(fromBool(value -> value ? 1f : 0f)),
STRING_TO_FLOAT(fromString(Float::valueOf, "Float")),
RATIONAL_TO_DOUBLE(fromDouble(value -> value)),
DATE_TO_FLOAT(fromDate(value -> (float) value)),
RATIONAL_TO_DOUBLE(fromDouble(Double::valueOf)),
INTEGER_TO_DOUBLE(fromLong(Double::valueOf)),
BOOL_TO_DOUBLE(fromBool(value -> value ? 1d : 0d)),
STRING_TO_DOUBLE(fromString(Double::valueOf, "Double")),
STRING_TO_DATE(fromString(UTC_DATE_FORMATTER::parseMillis, "Date")),
DATE_TO_DOUBLE(fromDate(Double::valueOf)),
RATIONAL_TO_DATE(toDate(RATIONAL_TO_LONG)),
INTEGER_TO_DATE(toDate(INTEGER_TO_LONG)),
BOOL_TO_DATE(toDate(BOOL_TO_INT)),
STRING_TO_DATE(fromString(UTC_DATE_FORMATTER::parseDateTime, "Date")),
NUMERIC_TO_BOOLEAN(fromLong(value -> value != 0)),
STRING_TO_BOOLEAN(fromString(DataTypeConversion::convertToBoolean, "Boolean"));
STRING_TO_BOOLEAN(fromString(DataTypeConversion::convertToBoolean, "Boolean")),
DATE_TO_BOOLEAN(fromDate(value -> value != 0));
private final Function<Object, Object> converter;
@ -359,7 +406,7 @@ public abstract class DataTypeConversion {
private static Function<Object, Object> fromLong(LongFunction<Object> converter) {
return (Object l) -> converter.apply(((Number) l).longValue());
}
private static Function<Object, Object> fromString(Function<String, Object> converter, String to) {
return (Object value) -> {
try {
@ -375,6 +422,14 @@ public abstract class DataTypeConversion {
private static Function<Object, Object> fromBool(Function<Boolean, Object> converter) {
return (Object l) -> converter.apply(((Boolean) l));
}
private static Function<Object, Object> fromDate(Function<Long, Object> converter) {
return l -> ((ReadableInstant) l).getMillis();
}
private static Function<Object, Object> toDate(Conversion conversion) {
return l -> new DateTime(((Number) conversion.convert(l)).longValue(), DateTimeZone.UTC);
}
public Object convert(Object l) {
if (l == null) {
@ -389,6 +444,6 @@ public abstract class DataTypeConversion {
return dataType;
}
return dataType.isInteger ? dataType : DataType.LONG;
return dataType.isInteger ? dataType : LONG;
}
}
}

55
plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java
View File

@ -23,10 +23,10 @@ public class DataTypeConversionTests extends ESTestCase {
}
/**
* Test conversion to a date or long. These are almost the same.
* Test conversion to long.
*/
public void testConversionToLongOrDate() {
DataType to = randomBoolean() ? DataType.LONG : DataType.DATE;
public void testConversionToLong() {
DataType to = DataType.LONG;
{
Conversion conversion = DataTypeConversion.conversionFor(DataType.DOUBLE, to);
assertNull(conversion.convert(null));
@ -50,19 +50,44 @@ public class DataTypeConversionTests extends ESTestCase {
}
Conversion conversion = DataTypeConversion.conversionFor(DataType.KEYWORD, to);
assertNull(conversion.convert(null));
if (to == DataType.LONG) {
assertEquals(1L, conversion.convert("1"));
assertEquals(0L, conversion.convert("-0"));
Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff"));
assertEquals("cannot cast [0xff] to [Long]", e.getMessage());
} else {
// TODO we'd like to be able to optionally parse millis here I think....
assertEquals(1000L, conversion.convert("1970-01-01T00:00:01Z"));
assertEquals(1483228800000L, conversion.convert("2017-01-01T00:00:00Z"));
assertEquals(18000000L, conversion.convert("1970-01-01T00:00:00-05:00"));
Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff"));
assertEquals("cannot cast [0xff] to [Date]:Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage());
assertEquals(1L, conversion.convert("1"));
assertEquals(0L, conversion.convert("-0"));
Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff"));
assertEquals("cannot cast [0xff] to [Long]", e.getMessage());
}
public void testConversionToDate() {
DataType to = DataType.DATE;
{
Conversion conversion = DataTypeConversion.conversionFor(DataType.DOUBLE, to);
assertNull(conversion.convert(null));
assertEquals(new DateTime(10L, DateTimeZone.UTC), conversion.convert(10.0));
assertEquals(new DateTime(10L, DateTimeZone.UTC), conversion.convert(10.1));
assertEquals(new DateTime(11L, DateTimeZone.UTC), conversion.convert(10.6));
Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert(Double.MAX_VALUE));
assertEquals("[" + Double.MAX_VALUE + "] out of [Long] range", e.getMessage());
}
{
Conversion conversion = DataTypeConversion.conversionFor(DataType.INTEGER, to);
assertNull(conversion.convert(null));
assertEquals(new DateTime(10L, DateTimeZone.UTC), conversion.convert(10));
assertEquals(new DateTime(-134L, DateTimeZone.UTC), conversion.convert(-134));
}
{
Conversion conversion = DataTypeConversion.conversionFor(DataType.BOOLEAN, to);
assertNull(conversion.convert(null));
assertEquals(new DateTime(1, DateTimeZone.UTC), conversion.convert(true));
assertEquals(new DateTime(0, DateTimeZone.UTC), conversion.convert(false));
}
Conversion conversion = DataTypeConversion.conversionFor(DataType.KEYWORD, to);
assertNull(conversion.convert(null));
// TODO we'd like to be able to optionally parse millis here I think....
assertEquals(new DateTime(1000L, DateTimeZone.UTC), conversion.convert("1970-01-01T00:00:01Z"));
assertEquals(new DateTime(1483228800000L, DateTimeZone.UTC), conversion.convert("2017-01-01T00:00:00Z"));
assertEquals(new DateTime(18000000L, DateTimeZone.UTC), conversion.convert("1970-01-01T00:00:00-05:00"));
Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff"));
assertEquals("cannot cast [0xff] to [Date]:Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage());
}
public void testConversionToDouble() {