SQL: Make parsing of date more lenient (#52137)

Make the parsing of date more lenient

- as an escaped literal: `{d '2020-02-10[[T| ]10:20[:30][.123456789][tz]]'}`
- cast a string to a date: `CAST(2020-02-10[[T| ]10:20[:30][.123456789][tz]]' AS DATE)`

Closes: #49379
(cherry picked from commit 5863b27500d5e7f6cdd8c6c62b09b84e53ca724a)

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java

@@ -130,9 +130,9 @@ import java.util.StringJoiner;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.xpack.ql.type.DataTypeConverter.converterFor;
-import static org.elasticsearch.xpack.sql.util.DateUtils.asDateOnly;
 import static org.elasticsearch.xpack.sql.util.DateUtils.asTimeOnly;
-import static org.elasticsearch.xpack.sql.util.DateUtils.ofEscapedLiteral;
+import static org.elasticsearch.xpack.sql.util.DateUtils.dateOfEscapedLiteral;
+import static org.elasticsearch.xpack.sql.util.DateUtils.dateTimeOfEscapedLiteral;
 
 abstract class ExpressionBuilder extends IdentifierBuilder {
 
@@ -761,9 +761,9 @@ abstract class ExpressionBuilder extends IdentifierBuilder {
     public Literal visitDateEscapedLiteral(DateEscapedLiteralContext ctx) {
         String string = string(ctx.string());
         Source source = source(ctx);
-        // parse yyyy-MM-dd
+        // parse yyyy-MM-dd (time optional but is set to 00:00:00.000 because of the conversion to DATE
         try {
-            return new Literal(source, asDateOnly(string), SqlDataTypes.DATE);
+            return new Literal(source, dateOfEscapedLiteral(string), SqlDataTypes.DATE);
         } catch(DateTimeParseException ex) {
             throw new ParsingException(source, "Invalid date received; {}", ex.getMessage());
         }
@@ -789,7 +789,7 @@ abstract class ExpressionBuilder extends IdentifierBuilder {
         Source source = source(ctx);
         // parse yyyy-mm-dd hh:mm:ss(.f...)
         try {
-            return new Literal(source, ofEscapedLiteral(string), DataTypes.DATETIME);
+            return new Literal(source, dateTimeOfEscapedLiteral(string), DataTypes.DATETIME);
         } catch (DateTimeParseException ex) {
             throw new ParsingException(source, "Invalid timestamp received; {}", ex.getMessage());
         }

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java

@@ -32,16 +32,40 @@ public final class DateUtils {
     public static final LocalDate EPOCH = LocalDate.of(1970, 1, 1);
     public static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000L;
 
-    private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
+    private static final DateTimeFormatter DATE_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
             .append(ISO_LOCAL_DATE)
             .appendLiteral(' ')
             .append(ISO_LOCAL_TIME)
             .toFormatter().withZone(UTC);
-    private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
+    private static final DateTimeFormatter DATE_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
             .append(ISO_LOCAL_DATE)
             .appendLiteral('T')
             .append(ISO_LOCAL_TIME)
             .toFormatter().withZone(UTC);
+    private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
+            .append(ISO_LOCAL_DATE)
+            .optionalStart()
+            .appendLiteral(' ')
+            .append(ISO_LOCAL_TIME)
+            .toFormatter().withZone(UTC);
+    private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
+            .append(ISO_LOCAL_DATE)
+            .optionalStart()
+            .appendLiteral('T')
+            .append(ISO_LOCAL_TIME)
+            .toFormatter().withZone(UTC);
+    private static final DateTimeFormatter ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
+            .append(DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE)
+            .optionalStart()
+            .appendZoneOrOffsetId()
+            .optionalEnd()
+            .toFormatter().withZone(UTC);
+    private static final DateTimeFormatter ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
+            .append(DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL)
+            .optionalStart()
+            .appendZoneOrOffsetId()
+            .optionalEnd()
+            .toFormatter().withZone(UTC);
 
     private static final DateFormatter UTC_DATE_TIME_FORMATTER = DateFormatter.forPattern("date_optional_time").withZone(UTC);
     private static final int DEFAULT_PRECISION_FOR_CURRENT_FUNCTIONS = 3;
@@ -91,7 +115,17 @@ public final class DateUtils {
      * Parses the given string into a Date (SQL DATE type) using UTC as a default timezone.
      */
     public static ZonedDateTime asDateOnly(String dateFormat) {
-        return LocalDate.parse(dateFormat, ISO_LOCAL_DATE).atStartOfDay(UTC);
+        int separatorIdx = dateFormat.indexOf('-');
+        if (separatorIdx == 0) { // negative year
+            separatorIdx = dateFormat.indexOf('-', 1);
+        }
+        separatorIdx = dateFormat.indexOf('-', separatorIdx + 1) + 3;
+        // Avoid index out of bounds - it will lead to DateTimeParseException anyways
+        if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
+            return LocalDate.parse(dateFormat, ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL).atStartOfDay(UTC);
+        } else {
+            return LocalDate.parse(dateFormat, ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE).atStartOfDay(UTC);
+        }
     }
 
     public static ZonedDateTime asDateOnly(ZonedDateTime zdt) {
@@ -109,13 +143,23 @@ public final class DateUtils {
         return DateFormatters.from(UTC_DATE_TIME_FORMATTER.parse(dateFormat)).withZoneSameInstant(UTC);
     }
 
-    public static ZonedDateTime ofEscapedLiteral(String dateFormat) {
+    public static ZonedDateTime dateOfEscapedLiteral(String dateFormat) {
         int separatorIdx = dateFormat.lastIndexOf('-') + 3;
         // Avoid index out of bounds - it will lead to DateTimeParseException anyways
         if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
-            return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER_T_LITERAL.withZone(UTC));
+            return LocalDate.parse(dateFormat, DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL).atStartOfDay(UTC);
         } else {
-            return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER_WHITESPACE.withZone(UTC));
+            return LocalDate.parse(dateFormat, DATE_TIME_FORMATTER_WHITESPACE).atStartOfDay(UTC);
+        }
+    }
+
+    public static ZonedDateTime dateTimeOfEscapedLiteral(String dateFormat) {
+        int separatorIdx = dateFormat.lastIndexOf('-') + 3;
+        // Avoid index out of bounds - it will lead to DateTimeParseException anyways
+        if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
+            return ZonedDateTime.parse(dateFormat, DATE_TIME_FORMATTER_T_LITERAL.withZone(UTC));
+        } else {
+            return ZonedDateTime.parse(dateFormat, DATE_TIME_FORMATTER_WHITESPACE.withZone(UTC));
         }
     }
 

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java

@@ -81,6 +81,13 @@ public class EscapedFunctionsTests extends ESTestCase {
         return sb.toString();
     }
 
+    private String buildTime() {
+        if (randomBoolean()) {
+            return (randomBoolean() ? "T" : " ") + "11:22" + buildSecsAndFractional();
+        }
+        return "";
+    }
+
     private String buildSecsAndFractional() {
         if (randomBoolean()) {
             return ":55" + randomFrom("", ".1", ".12", ".123", ".1234", ".12345", ".123456",
@@ -212,7 +219,7 @@ public class EscapedFunctionsTests extends ESTestCase {
     }
 
     public void testDateLiteral() {
-        Literal l = dateLiteral(buildDate());
+        Literal l = dateLiteral(buildDate() + buildTime());
         assertThat(l.dataType(), is(DATE));
     }
 

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java

@@ -172,19 +172,37 @@ public class SqlDataTypeConverterTests extends ESTestCase {
             Converter conversion = converterFor(KEYWORD, to);
             assertNull(conversion.convert(null));
 
-            assertEquals(date(0L), conversion.convert("1970-01-01"));
+            assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20"));
-            assertEquals(date(1483228800000L), conversion.convert("2017-01-01"));
+            assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10T10:20:30.123"));
-            assertEquals(date(-1672531200000L), conversion.convert("1917-01-01"));
+            assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20:30.123456789"));
-            assertEquals(date(18000000L), conversion.convert("1970-01-01"));
+
+            assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20"));
+            assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10 10:20:30.123"));
+            assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20:30.123456789"));
+
+            assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20+05:00"));
+            assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10T10:20:30.123-06:00"));
+            assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20:30.123456789+03:00"));
+
+            assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20+05:00"));
+            assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10 10:20:30.123-06:00"));
+            assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20:30.123456789+03:00"));
 
             // double check back and forth conversion
-
             ZonedDateTime zdt = org.elasticsearch.common.time.DateUtils.nowWithMillisResolution();
             Converter forward = converterFor(DATE, KEYWORD);
             Converter back = converterFor(KEYWORD, DATE);
             assertEquals(asDateOnly(zdt), back.convert(forward.convert(zdt)));
             Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("0xff"));
             assertEquals("cannot cast [0xff] to [date]: Text '0xff' could not be parsed at index 0", e.getMessage());
+            e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("2020-02-"));
+            assertEquals("cannot cast [2020-02-] to [date]: Text '2020-02-' could not be parsed at index 8", e.getMessage());
+            e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("2020-"));
+            assertEquals("cannot cast [2020-] to [date]: Text '2020-' could not be parsed at index 5", e.getMessage());
+            e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("-2020-02-"));
+            assertEquals("cannot cast [-2020-02-] to [date]: Text '-2020-02-' could not be parsed at index 9", e.getMessage());
+            e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("-2020-"));
+            assertEquals("cannot cast [-2020-] to [date]: Text '-2020-' could not be parsed at index 6", e.getMessage());
         }
     }
 
@@ -285,7 +303,6 @@ public class SqlDataTypeConverterTests extends ESTestCase {
             assertEquals(dateTime(18000000L), conversion.convert("1970-01-01T00:00:00-05:00"));
 
             // double check back and forth conversion
-
             ZonedDateTime dt = org.elasticsearch.common.time.DateUtils.nowWithMillisResolution();
             Converter forward = converterFor(DATETIME, KEYWORD);
             Converter back = converterFor(KEYWORD, DATETIME);
@@ -692,4 +709,4 @@ public class SqlDataTypeConverterTests extends ESTestCase {
     static OffsetTime time(long millisSinceEpoch) {
         return DateUtils.asTimeOnly(millisSinceEpoch);
     }
-}
+}