honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SQL: Make parsing of date more lenient (#52137) 

Make the parsing of date more lenient

- as an escaped literal: `{d '2020-02-10[[T| ]10:20[:30][.123456789][tz]]'}`
- cast a string to a date: `CAST(2020-02-10[[T| ]10:20[:30][.123456789][tz]]' AS DATE)`

Closes: #49379
(cherry picked from commit 5863b27500d5e7f6cdd8c6c62b09b84e53ca724a)
This commit is contained in:
Marios Trivyzas 2020-02-10 21:45:06 +01:00
parent 47255c4fd7
commit 6b600855a9
No known key found for this signature in database
GPG Key ID: 8817B46B0CF36A3F
4 changed files with 87 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
View File

@ -130,9 +130,9 @@ import java.util.StringJoiner;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.ql.type.DataTypeConverter.converterFor;
import static org.elasticsearch.xpack.sql.util.DateUtils.asDateOnly;
import static org.elasticsearch.xpack.sql.util.DateUtils.asTimeOnly;
import static org.elasticsearch.xpack.sql.util.DateUtils.ofEscapedLiteral;
import static org.elasticsearch.xpack.sql.util.DateUtils.dateOfEscapedLiteral;
import static org.elasticsearch.xpack.sql.util.DateUtils.dateTimeOfEscapedLiteral;
abstract class ExpressionBuilder extends IdentifierBuilder {
@ -761,9 +761,9 @@ abstract class ExpressionBuilder extends IdentifierBuilder {
public Literal visitDateEscapedLiteral(DateEscapedLiteralContext ctx) {
String string = string(ctx.string());
Source source = source(ctx);
// parse yyyy-MM-dd
// parse yyyy-MM-dd (time optional but is set to 00:00:00.000 because of the conversion to DATE
try {
return new Literal(source, asDateOnly(string), SqlDataTypes.DATE);
return new Literal(source, dateOfEscapedLiteral(string), SqlDataTypes.DATE);
} catch(DateTimeParseException ex) {
throw new ParsingException(source, "Invalid date received; {}", ex.getMessage());
}
@ -789,7 +789,7 @@ abstract class ExpressionBuilder extends IdentifierBuilder {
Source source = source(ctx);
// parse yyyy-mm-dd hh:mm:ss(.f...)
try {
return new Literal(source, ofEscapedLiteral(string), DataTypes.DATETIME);
return new Literal(source, dateTimeOfEscapedLiteral(string), DataTypes.DATETIME);
} catch (DateTimeParseException ex) {
throw new ParsingException(source, "Invalid timestamp received; {}", ex.getMessage());
}

56
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java
View File

@ -32,16 +32,40 @@ public final class DateUtils {
public static final LocalDate EPOCH = LocalDate.of(1970, 1, 1);
public static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000L;
private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
private static final DateTimeFormatter DATE_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
.append(ISO_LOCAL_DATE)
.appendLiteral(' ')
.append(ISO_LOCAL_TIME)
.toFormatter().withZone(UTC);
private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
private static final DateTimeFormatter DATE_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
.append(ISO_LOCAL_DATE)
.appendLiteral('T')
.append(ISO_LOCAL_TIME)
.toFormatter().withZone(UTC);
private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
.append(ISO_LOCAL_DATE)
.optionalStart()
.appendLiteral(' ')
.append(ISO_LOCAL_TIME)
.toFormatter().withZone(UTC);
private static final DateTimeFormatter DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
.append(ISO_LOCAL_DATE)
.optionalStart()
.appendLiteral('T')
.append(ISO_LOCAL_TIME)
.toFormatter().withZone(UTC);
private static final DateTimeFormatter ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE = new DateTimeFormatterBuilder()
.append(DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE)
.optionalStart()
.appendZoneOrOffsetId()
.optionalEnd()
.toFormatter().withZone(UTC);
private static final DateTimeFormatter ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL = new DateTimeFormatterBuilder()
.append(DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL)
.optionalStart()
.appendZoneOrOffsetId()
.optionalEnd()
.toFormatter().withZone(UTC);
private static final DateFormatter UTC_DATE_TIME_FORMATTER = DateFormatter.forPattern("date_optional_time").withZone(UTC);
private static final int DEFAULT_PRECISION_FOR_CURRENT_FUNCTIONS = 3;
@ -91,7 +115,17 @@ public final class DateUtils {
* Parses the given string into a Date (SQL DATE type) using UTC as a default timezone.
*/
public static ZonedDateTime asDateOnly(String dateFormat) {
return LocalDate.parse(dateFormat, ISO_LOCAL_DATE).atStartOfDay(UTC);
int separatorIdx = dateFormat.indexOf('-');
if (separatorIdx == 0) { // negative year
separatorIdx = dateFormat.indexOf('-', 1);
}
separatorIdx = dateFormat.indexOf('-', separatorIdx + 1) + 3;
// Avoid index out of bounds - it will lead to DateTimeParseException anyways
if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
return LocalDate.parse(dateFormat, ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL).atStartOfDay(UTC);
} else {
return LocalDate.parse(dateFormat, ISO_LOCAL_DATE_OPTIONAL_TIME_FORMATTER_WHITESPACE).atStartOfDay(UTC);
}
}
public static ZonedDateTime asDateOnly(ZonedDateTime zdt) {
@ -109,13 +143,23 @@ public final class DateUtils {
return DateFormatters.from(UTC_DATE_TIME_FORMATTER.parse(dateFormat)).withZoneSameInstant(UTC);
}
public static ZonedDateTime ofEscapedLiteral(String dateFormat) {
public static ZonedDateTime dateOfEscapedLiteral(String dateFormat) {
int separatorIdx = dateFormat.lastIndexOf('-') + 3;
// Avoid index out of bounds - it will lead to DateTimeParseException anyways
if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER_T_LITERAL.withZone(UTC));
return LocalDate.parse(dateFormat, DATE_OPTIONAL_TIME_FORMATTER_T_LITERAL).atStartOfDay(UTC);
} else {
return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER_WHITESPACE.withZone(UTC));
return LocalDate.parse(dateFormat, DATE_TIME_FORMATTER_WHITESPACE).atStartOfDay(UTC);
}
}
public static ZonedDateTime dateTimeOfEscapedLiteral(String dateFormat) {
int separatorIdx = dateFormat.lastIndexOf('-') + 3;
// Avoid index out of bounds - it will lead to DateTimeParseException anyways
if (separatorIdx >= dateFormat.length() || dateFormat.charAt(separatorIdx) == 'T') {
return ZonedDateTime.parse(dateFormat, DATE_TIME_FORMATTER_T_LITERAL.withZone(UTC));
} else {
return ZonedDateTime.parse(dateFormat, DATE_TIME_FORMATTER_WHITESPACE.withZone(UTC));
}
}

9
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java
View File

@ -81,6 +81,13 @@ public class EscapedFunctionsTests extends ESTestCase {
return sb.toString();
}
private String buildTime() {
if (randomBoolean()) {
return (randomBoolean() ? "T" : " ") + "11:22" + buildSecsAndFractional();
}
return "";
}
private String buildSecsAndFractional() {
if (randomBoolean()) {
return ":55" + randomFrom("", ".1", ".12", ".123", ".1234", ".12345", ".123456",
@ -212,7 +219,7 @@ public class EscapedFunctionsTests extends ESTestCase {
}
public void testDateLiteral() {
Literal l = dateLiteral(buildDate());
Literal l = dateLiteral(buildDate() + buildTime());
assertThat(l.dataType(), is(DATE));
}

31
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/SqlDataTypeConverterTests.java
View File

@ -172,19 +172,37 @@ public class SqlDataTypeConverterTests extends ESTestCase {
Converter conversion = converterFor(KEYWORD, to);
assertNull(conversion.convert(null));
assertEquals(date(0L), conversion.convert("1970-01-01"));
assertEquals(date(1483228800000L), conversion.convert("2017-01-01"));
assertEquals(date(-1672531200000L), conversion.convert("1917-01-01"));
assertEquals(date(18000000L), conversion.convert("1970-01-01"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20"));
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10T10:20:30.123"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20:30.123456789"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20"));
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10 10:20:30.123"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20:30.123456789"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20+05:00"));
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10T10:20:30.123-06:00"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10T10:20:30.123456789+03:00"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20+05:00"));
assertEquals(date(-125908819200000L), conversion.convert("-2020-02-10 10:20:30.123-06:00"));
assertEquals(date(1581292800000L), conversion.convert("2020-02-10 10:20:30.123456789+03:00"));
// double check back and forth conversion
ZonedDateTime zdt = org.elasticsearch.common.time.DateUtils.nowWithMillisResolution();
Converter forward = converterFor(DATE, KEYWORD);
Converter back = converterFor(KEYWORD, DATE);
assertEquals(asDateOnly(zdt), back.convert(forward.convert(zdt)));
Exception e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("0xff"));
assertEquals("cannot cast [0xff] to [date]: Text '0xff' could not be parsed at index 0", e.getMessage());
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("2020-02-"));
assertEquals("cannot cast [2020-02-] to [date]: Text '2020-02-' could not be parsed at index 8", e.getMessage());
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("2020-"));
assertEquals("cannot cast [2020-] to [date]: Text '2020-' could not be parsed at index 5", e.getMessage());
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("-2020-02-"));
assertEquals("cannot cast [-2020-02-] to [date]: Text '-2020-02-' could not be parsed at index 9", e.getMessage());
e = expectThrows(QlIllegalArgumentException.class, () -> conversion.convert("-2020-"));
assertEquals("cannot cast [-2020-] to [date]: Text '-2020-' could not be parsed at index 6", e.getMessage());
}
}
@ -285,7 +303,6 @@ public class SqlDataTypeConverterTests extends ESTestCase {
assertEquals(dateTime(18000000L), conversion.convert("1970-01-01T00:00:00-05:00"));
// double check back and forth conversion
ZonedDateTime dt = org.elasticsearch.common.time.DateUtils.nowWithMillisResolution();
Converter forward = converterFor(DATETIME, KEYWORD);
Converter back = converterFor(KEYWORD, DATETIME);
@ -692,4 +709,4 @@ public class SqlDataTypeConverterTests extends ESTestCase {
static OffsetTime time(long millisSinceEpoch) {
return DateUtils.asTimeOnly(millisSinceEpoch);
}
}
}