From 6b600855a96c23b4591da75edd84f7078db263bd Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Mon, 10 Feb 2020 21:45:06 +0100 Subject: [PATCH] SQL: Make parsing of date more lenient (#52137) Make the parsing of date more lenient - as an escaped literal: `{d '2020-02-10[[T| ]10:20[:30][.123456789][tz]]'}` - cast a string to a date: `CAST(2020-02-10[[T| ]10:20[:30][.123456789][tz]]' AS DATE)` Closes: #49379 (cherry picked from commit 5863b27500d5e7f6cdd8c6c62b09b84e53ca724a) --- .../xpack/sql/parser/ExpressionBuilder.java | 10 ++-- .../xpack/sql/util/DateUtils.java | 56 +++++++++++++++++-- .../sql/parser/EscapedFunctionsTests.java | 9 ++- .../sql/type/SqlDataTypeConverterTests.java | 31 +++++++--- 4 files changed, 87 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index df9453228c7..fdb0f554ccd 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -130,9 +130,9 @@ import java.util.StringJoiner; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.ql.type.DataTypeConverter.converterFor; -import static org.elasticsearch.xpack.sql.util.DateUtils.asDateOnly; import static org.elasticsearch.xpack.sql.util.DateUtils.asTimeOnly; -import static org.elasticsearch.xpack.sql.util.DateUtils.ofEscapedLiteral; +import static org.elasticsearch.xpack.sql.util.DateUtils.dateOfEscapedLiteral; +import static org.elasticsearch.xpack.sql.util.DateUtils.dateTimeOfEscapedLiteral; abstract class ExpressionBuilder extends IdentifierBuilder { @@ -761,9 +761,9 @@ abstract class ExpressionBuilder extends IdentifierBuilder { public Literal visitDateEscapedLiteral(DateEscapedLiteralContext ctx) { String string = string(ctx.string()); Source source = source(ctx); - // parse yyyy-MM-dd + // parse yyyy-MM-dd (time optional but is set to 00:00:00.000 because of the conversion to DATE try { - return new Literal(source, asDateOnly(string), SqlDataTypes.DATE); + return new Literal(source, dateOfEscapedLiteral(string), SqlDataTypes.DATE); } catch(DateTimeParseException ex) { throw new ParsingException(source, "Invalid date received; {}", ex.getMessage()); } @@ -789,7 +789,7 @@ abstract class ExpressionBuilder extends IdentifierBuilder { Source source = source(ctx); // parse yyyy-mm-dd hh:mm:ss(.f...) try { - return new Literal(source, ofEscapedLiteral(string), DataTypes.DATETIME); + return new Literal(source, dateTimeOfEscapedLiteral(string), DataTypes.DATETIME); } catch (DateTimeParseException ex) { throw new ParsingException(source, "Invalid timestamp received; {}", ex.getMessage()); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java index 31c79f7a1bc..7ea20224a86 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java @@ -32,16 +32,40 @@ public final class DateUtils { public static final LocalDate EPOCH = LocalDate.of(1970, 1, 1); public static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000L; - private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder() + private static final DateTimeFormatter DATE_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder() .append(ISO_LOCAL_DATE) .appendLiteral(' ') .append(ISO_LOCAL_TIME) .toFormatter().withZone(UTC); - private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder() + private static final DateTimeFormatter DATE_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder() .append(ISO_LOCAL_DATE) .appendLiteral('T') .append(ISO_LOCAL_TIME) .toFormatter().withZone(UTC); + private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder() + .append(ISO_LOCAL_DATE) + .optionalStart() + .appendLiteral(' ') + .append(ISO_LOCAL_TIME) + .toFormatter().withZone(UTC); + private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder() + .append(ISO_LOCAL_DATE) + .optionalStart() + .appendLiteral('T') + .append(ISO_LOCAL_TIME) + .toFormatter().withZone(UTC); + private static final DateTimeFormatter ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder() + .append(DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE) + .optionalStart() + .appendZoneOrOffsetId() + .optionalEnd() + .toFormatter().withZone(UTC); + private static final DateTimeFormatter ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder() + .append(DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL) + .optionalStart() + .appendZoneOrOffsetId() + .optionalEnd() + .toFormatter().withZone(UTC); private static final DateFormatter UTC_DATE_TIME_FORMATTER = DateFormatter.forPattern("date_optional_time").withZone(UTC); private static final int DEFAULT_PRECISION_FOR_CURRENT_FUNCTIONS = 3; @@ -91,7 +115,17 @@ public final class DateUtils { * Parses the given string into a Date (SQL DATE type) using UTC as a default timezone. */ public static ZonedDateTime asDateOnly(String dateFormat) { - return LocalDate.parse(dateFormat, ISO_LOCAL_DATE).atStartOfDay(UTC); + int separatorIdx = dateFormat.indexOf('-'); + if (separatorIdx == 0) { // negative year + separatorIdx = dateFormat.indexOf('-', 1); + } + separatorIdx = dateFormat.indexOf('-', separatorIdx + 1) + 3; + // Avoid index out of bounds - it will lead to DateTimeParseException anyways + if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') { + return LocalDate.parse(dateFormat, ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL).atStartOfDay(UTC); + } else { + return LocalDate.parse(dateFormat, ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE).atStartOfDay(UTC); + } } public static ZonedDateTime asDateOnly(ZonedDateTime zdt) { @@ -109,13 +143,23 @@ public final class DateUtils { return DateFormatters.from(UTC_DATE_TIME_FORMATTER.parse(dateFormat)).withZoneSameInstant(UTC); } - public static ZonedDateTime ofEscapedLiteral(String dateFormat) { + public static ZonedDateTime dateOfEscapedLiteral(String dateFormat) { int separatorIdx = dateFormat.lastIndexOf('-') + 3; // Avoid index out of bounds - it will lead to DateTimeParseException anyways if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') { - return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER_T_LITERAL.withZone(UTC)); + return LocalDate.parse(dateFormat, DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL).atStartOfDay(UTC); } else { - return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER_WHITESPACE.withZone(UTC)); + return LocalDate.parse(dateFormat, DATE_TIME_FORMATTER_WHITESPACE).atStartOfDay(UTC); + } + } + + public static ZonedDateTime dateTimeOfEscapedLiteral(String dateFormat) { + int separatorIdx = dateFormat.lastIndexOf('-') + 3; + // Avoid index out of bounds - it will lead to DateTimeParseException anyways + if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') { + return ZonedDateTime.parse(dateFormat, DATE_TIME_FORMATTER_T_LITERAL.withZone(UTC)); + } else { + return ZonedDateTime.parse(dateFormat, DATE_TIME_FORMATTER_WHITESPACE.withZone(UTC)); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java index 93395f9dbd3..1a05f90791c 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java @@ -81,6 +81,13 @@ public class EscapedFunctionsTests extends ESTestCase { return sb.toString(); } + private String buildTime() { + if (randomBoolean()) { + return (randomBoolean() ? "T" : " ") + "11:22" + buildSecsAndFractional(); + } + return ""; + } + private String buildSecsAndFractional() { if (randomBoolean()) { return ":55" + randomFrom("", ".1", ".12", ".123", ".1234", ".12345", ".123456", @@ -212,7 +219,7 @@ public class EscapedFunctionsTests extends ESTestCase { } public void testDateLiteral() { - Literal l = dateLiteral(buildDate()); + Literal l = dateLiteral(buildDate() + buildTime()); assertThat(l.dataType(), is(DATE)); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java index 9a2db21d6ec..b39dd8bcaa8 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java @@ -172,19 +172,37 @@ public class SqlDataTypeConverterTests extends ESTestCase { Converter conversion = converterFor(KEYWORD, to); assertNull(conversion.convert(null)); - assertEquals(date(0L), conversion.convert("1970-01-01")); - assertEquals(date(1483228800000L), conversion.convert("2017-01-01")); - assertEquals(date(-1672531200000L), conversion.convert("1917-01-01")); - assertEquals(date(18000000L), conversion.convert("1970-01-01")); + assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20")); + assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10T10:20:30.123")); + assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20:30.123456789")); + + assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20")); + assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10 10:20:30.123")); + assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20:30.123456789")); + + assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20+05:00")); + assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10T10:20:30.123-06:00")); + assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20:30.123456789+03:00")); + + assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20+05:00")); + assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10 10:20:30.123-06:00")); + assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20:30.123456789+03:00")); // double check back and forth conversion - ZonedDateTime zdt = org.elasticsearch.common.time.DateUtils.nowWithMillisResolution(); Converter forward = converterFor(DATE, KEYWORD); Converter back = converterFor(KEYWORD, DATE); assertEquals(asDateOnly(zdt), back.convert(forward.convert(zdt))); Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("0xff")); assertEquals("cannot cast [0xff] to [date]: Text '0xff' could not be parsed at index 0", e.getMessage()); + e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("2020-02-")); + assertEquals("cannot cast [2020-02-] to [date]: Text '2020-02-' could not be parsed at index 8", e.getMessage()); + e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("2020-")); + assertEquals("cannot cast [2020-] to [date]: Text '2020-' could not be parsed at index 5", e.getMessage()); + e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("-2020-02-")); + assertEquals("cannot cast [-2020-02-] to [date]: Text '-2020-02-' could not be parsed at index 9", e.getMessage()); + e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("-2020-")); + assertEquals("cannot cast [-2020-] to [date]: Text '-2020-' could not be parsed at index 6", e.getMessage()); } } @@ -285,7 +303,6 @@ public class SqlDataTypeConverterTests extends ESTestCase { assertEquals(dateTime(18000000L), conversion.convert("1970-01-01T00:00:00-05:00")); // double check back and forth conversion - ZonedDateTime dt = org.elasticsearch.common.time.DateUtils.nowWithMillisResolution(); Converter forward = converterFor(DATETIME, KEYWORD); Converter back = converterFor(KEYWORD, DATETIME); @@ -692,4 +709,4 @@ public class SqlDataTypeConverterTests extends ESTestCase { static OffsetTime time(long millisSinceEpoch) { return DateUtils.asTimeOnly(millisSinceEpoch); } -} \ No newline at end of file +}