honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SQL: Implement DATETIME_PARSE function for parsing strings (#54960) (#55035) 

Implement DATETIME_PARSE(<datetime_str>, <pattern_str>) function
which allows to parse a datetime string according to the specified
pattern into a datetime object. The patterns allowed are those of
java.time.format.DateTimeFormatter.

Relates to #53714

(cherry picked from commit 3febcd8f3cdf9fdda4faf01f23a5f139f38b57e0)
This commit is contained in:
Marios Trivyzas 2020-04-10 01:16:29 +02:00 committed by GitHub
parent 51cb0c5c7b
commit bf0cadb602
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 672 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
docs/reference/sql/functions/date-time.asciidoc
View File

@ -427,7 +427,7 @@ pattern used is the one from
https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/time/format/DateTimeFormatter.html[`java.time.format.DateTimeFormatter`].
If any of the two arguments is `null` or the pattern is an empty string `null` is returned.
NOTE::
[NOTE]
If the 1st argument is of type `time`, then pattern specified by the 2nd argument cannot contain date related units
(e.g. 'dd', 'MM', 'YYYY', etc.). If it contains such units an error is returned.
@ -446,6 +446,47 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[dateTimeFormatDateTime]
include-tagged::{sql-specs}/docs/docs.csv-spec[dateTimeFormatTime]
--------------------------------------------------
[[sql-functions-datetime-datetimeparse]]
==== `DATETIME_PARSE`
.Synopsis:
[source, sql]
--------------------------------------------------
DATETIME_PARSE(
string_exp, <1>
string_exp) <2>
--------------------------------------------------
*Input*:
<1> datetime expression as a string
<2> parsing pattern
*Output*: datetime
*Description*: Returns a datetime by parsing the 1st argument using the format specified in the 2nd argument. The parsing
format pattern used is the one from
https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/time/format/DateTimeFormatter.html[`java.time.format.DateTimeFormatter`].
If any of the two arguments is `null` or an empty string `null` is returned.
[NOTE]
If timezone is not specified in the datetime string expression and the parsing pattern, the resulting `datetime` will
be in `UTC` timezone.
[NOTE]
If the parsing pattern contains only date or only time units (e.g. 'dd/MM/uuuu', 'HH:mm:ss', etc.) an error is returned
as the function needs to return a value of `datetime` type which must contain both.
[source, sql]
--------------------------------------------------
include-tagged::{sql-specs}/docs/docs.csv-spec[dateTimeParse1]
--------------------------------------------------
[source, sql]
--------------------------------------------------
include-tagged::{sql-specs}/docs/docs.csv-spec[dateTimeParse2]
--------------------------------------------------
[[sql-functions-datetime-part]]
==== `DATE_PART/DATEPART`

1
docs/reference/sql/functions/index.asciidoc
View File

@ -54,6 +54,7 @@
** <<sql-functions-datetime-add>>
** <<sql-functions-datetime-diff>>
** <<sql-functions-datetime-datetimeformat>>
** <<sql-functions-datetime-datetimeparse>>
** <<sql-functions-datetime-part>>
** <<sql-functions-datetime-trunc>>
** <<sql-functions-datetime-day>>

1
x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
View File

@ -45,6 +45,7 @@ DATEADD |SCALAR
DATEDIFF |SCALAR
DATEPART |SCALAR
DATETIME_FORMAT |SCALAR
DATETIME_PARSE |SCALAR
DATETRUNC |SCALAR
DATE_ADD |SCALAR
DATE_DIFF |SCALAR

102
x-pack/plugin/sql/qa/src/main/resources/datetime.csv-spec
View File

@ -580,6 +580,108 @@ HAVING DATETIME_FORMAT(MAX(birth_date), 'dd')::integer > 20 ORDER BY 1 DESC;
1961-02-26 00:00:00.000Z | 02
;
selectDateTimeParse
schema::dp_date1:ts|dp_date2:ts
SELECT DATETIME_PARSE('07/04/2020___11:22:33 Europe/Berlin', 'dd/MM/uuuu___HH:mm:ss VV') AS dp_date1,
DATETIME_PARSE('11:22:33 2020__04__07 -05:33', 'HH:mm:ss uuuu__MM__dd zz') AS dp_date2;
dp_date1 | dp_date2
----------------------------+----------------------------
2020-04-07T09:22:33:00.000Z | 2020-04-07T16:55:33:00.000Z
;
selectDateTimeParseWithField
schema::birth_date:ts|dp_birth_date:ts
SELECT birth_date, DATETIME_PARSE(DATETIME_FORMAT(birth_date, 'MM/dd/ HH uuuu mm SSS ss'), concat(gender, 'M/dd/ HH uuuu mm SSS ss')) AS dp_birth_date
FROM test_emp WHERE gender = 'M' AND emp_no BETWEEN 10037 AND 10052 ORDER BY emp_no;
birth_date | dp_birth_date
-------------------------+-------------------------
1963-07-22 00:00:00.000Z | 1963-07-22 00:00:00.000Z
1960-07-20 00:00:00.000Z | 1960-07-20 00:00:00.000Z
1959-10-01 00:00:00.000Z | 1959-10-01 00:00:00.000Z
null | null
null | null
null | null
null | null
null | null
1958-05-21 00:00:00.000Z | 1958-05-21 00:00:00.000Z
1953-07-28 00:00:00.000Z | 1953-07-28 00:00:00.000Z
1961-02-26 00:00:00.000Z | 1961-02-26 00:00:00.000Z
;
dateTimeParseWhere
schema::birth_date:ts|dp_birth_date:ts
SELECT birth_date, DATETIME_PARSE(DATETIME_FORMAT(birth_date, 'MM_dd_uuuu HH.mm.ss:SSS'), 'MM_dd_uuuu HH.mm.ss:SSS') AS dp_birth_date
FROM test_emp WHERE dp_birth_date > '1963-10-20'::date ORDER BY emp_no;
birth_date | dp_birth_date
-------------------------+------------------------
1964-06-02 00:00:00.000Z | 1964-06-02 00:00:00.000Z
1963-11-26 00:00:00.000Z | 1963-11-26 00:00:00.000Z
1964-04-18 00:00:00.000Z | 1964-04-18 00:00:00.000Z
1964-10-18 00:00:00.000Z | 1964-10-18 00:00:00.000Z
1964-06-11 00:00:00.000Z | 1964-06-11 00:00:00.000Z
1965-01-03 00:00:00.000Z | 1965-01-03 00:00:00.000Z
;
dateTimeParseOrderBy
schema::birth_date:ts|dp_birth_date:ts
SELECT birth_date, DATETIME_PARSE(DATETIME_FORMAT(birth_date, 'HH:mm:ss.SSS MM/dd/uuuu'), 'HH:mm:ss.SSS MM/dd/uuuu') AS dp_birth_date
FROM test_emp ORDER BY 2 DESC NULLS LAST LIMIT 10;
birth_date | dp_birth_date
-------------------------+-------------------------
1965-01-03 00:00:00.000Z | 1965-01-03 00:00:00.000Z
1964-10-18 00:00:00.000Z | 1964-10-18 00:00:00.000Z
1964-06-11 00:00:00.000Z | 1964-06-11 00:00:00.000Z
1964-06-02 00:00:00.000Z | 1964-06-02 00:00:00.000Z
1964-04-18 00:00:00.000Z | 1964-04-18 00:00:00.000Z
1963-11-26 00:00:00.000Z | 1963-11-26 00:00:00.000Z
1963-09-09 00:00:00.000Z | 1963-09-09 00:00:00.000Z
1963-07-22 00:00:00.000Z | 1963-07-22 00:00:00.000Z
1963-06-07 00:00:00.000Z | 1963-06-07 00:00:00.000Z
1963-06-01 00:00:00.000Z | 1963-06-01 00:00:00.000Z
;
dateTimeParseGroupBy
schema::count:l|df_birth_date:s
SELECT count(*) AS count, DATETIME_FORMAT(DATETIME_PARSE(DATETIME_FORMAT(birth_date, 'dd/MM/uuuu HH:mm:ss'), 'dd/MM/uuuu HH:mm:ss'), 'MM') AS df_birth_date
FROM test_emp GROUP BY df_birth_date ORDER BY 1 DESC, 2 DESC NULLS LAST LIMIT 10;
count | df_birth_date
-------+---------------
10 | 09
10 | 05
10 | null
9 | 10
9 | 07
8 | 11
8 | 04
8 | 02
7 | 12
7 | 06
;
dateTimeParseHaving
schema::max:ts|df_birth_date:s
SELECT MAX(birth_date) AS max, DATETIME_FORMAT(birth_date, 'MM') AS df_birth_date FROM test_emp GROUP BY df_birth_date
HAVING DATETIME_PARSE(DATETIME_FORMAT(MAX(birth_date), 'dd/MM/uuuu HH:mm:ss'), 'dd/MM/uuuu HH:mm:ss') > '1961-10-20'::date ORDER BY 1 DESC NULLS LAST;
max | df_birth_date
-------------------------+---------------
1965-01-03 00:00:00.000Z | 01
1964-10-18 00:00:00.000Z | 10
1964-06-11 00:00:00.000Z | 06
1964-04-18 00:00:00.000Z | 04
1963-11-26 00:00:00.000Z | 11
1963-09-09 00:00:00.000Z | 09
1963-07-22 00:00:00.000Z | 07
1963-03-21 00:00:00.000Z | 03
1962-12-29 00:00:00.000Z | 12
null | null
;
selectDateTruncWithDateTime
schema::dt_hour:ts|dt_min:ts|dt_sec:ts|dt_millis:s|dt_micro:s|dt_nano:s
SELECT DATE_TRUNC('hour', '2019-09-04T11:22:33.123Z'::datetime) as dt_hour, DATE_TRUNC('minute', '2019-09-04T11:22:33.123Z'::datetime) as dt_min,

24
x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec
View File

@ -241,6 +241,7 @@ DATEADD |SCALAR
DATEDIFF |SCALAR
DATEPART |SCALAR
DATETIME_FORMAT |SCALAR
DATETIME_PARSE |SCALAR
DATETRUNC |SCALAR
DATE_ADD |SCALAR
DATE_DIFF |SCALAR
@ -2578,6 +2579,29 @@ SELECT DATETIME_FORMAT(CAST('11:22:33.987' AS TIME), 'HH mm ss.S') AS "time";
// end::dateTimeFormatTime
;
// Cannot assert millis: https://github.com/elastic/elasticsearch/issues/54947
dateTimeParse1-Ignore
schema::datetime:ts
// tag::dateTimeParse1
SELECT DATETIME_PARSE('07/04/2020 10:20:30.123', 'dd/MM/uuuu HH:mm:ss.SSS') AS "datetime";
datetime
------------------------
2020-04-07T10:20:30.123Z
// end::dateTimeParse1
;
dateTimeParse2
schema::datetime:ts
// tag::dateTimeParse2
SELECT DATETIME_PARSE('10:20:30 07/04/2020 Europe/Berlin', 'HH:mm:ss dd/MM/uuuu VV') AS "datetime";
datetime
------------------------
2020-04-07T08:20:30.000Z
// end::dateTimeParse2
;
datePartDateTimeYears
// tag::datePartDateTimeYears
SELECT DATE_PART('year', '2019-09-22T11:22:33.123Z'::datetime) AS "years";

2
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/SqlFunctionRegistry.java
View File

@ -33,6 +33,7 @@ import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateAdd;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateDiff;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DatePart;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFormat;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeParse;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTrunc;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DayName;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DayOfMonth;
@ -170,6 +171,7 @@ public class SqlFunctionRegistry extends FunctionRegistry {
def(DateDiff.class, DateDiff::new, "DATEDIFF", "DATE_DIFF", "TIMESTAMPDIFF", "TIMESTAMP_DIFF"),
def(DatePart.class, DatePart::new, "DATEPART", "DATE_PART"),
def(DateTimeFormat.class, DateTimeFormat::new, "DATETIME_FORMAT"),
def(DateTimeParse.class, DateTimeParse::new, "DATETIME_PARSE"),
def(DateTrunc.class, DateTrunc::new, "DATETRUNC", "DATE_TRUNC"),
def(HourOfDay.class, HourOfDay::new, "HOUR_OF_DAY", "HOUR"),
def(IsoDayOfWeek.class, IsoDayOfWeek::new, "ISO_DAY_OF_WEEK", "ISODAYOFWEEK", "ISODOW", "IDOW"),

4
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java
View File

@ -16,6 +16,7 @@ import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateAddPr
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateDiffProcessor;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DatePartProcessor;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFormatProcessor;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeParseProcessor;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeProcessor;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTruncProcessor;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.NamedDateTimeProcessor;
@ -84,8 +85,9 @@ public final class Processors {
entries.add(new Entry(Processor.class, DateAddProcessor.NAME, DateAddProcessor::new));
entries.add(new Entry(Processor.class, DateDiffProcessor.NAME, DateDiffProcessor::new));
entries.add(new Entry(Processor.class, DatePartProcessor.NAME, DatePartProcessor::new));
entries.add(new Entry(Processor.class, DateTruncProcessor.NAME, DateTruncProcessor::new));
entries.add(new Entry(Processor.class, DateTimeFormatProcessor.NAME, DateTimeFormatProcessor::new));
entries.add(new Entry(Processor.class, DateTimeParseProcessor.NAME, DateTimeParseProcessor::new));
entries.add(new Entry(Processor.class, DateTruncProcessor.NAME, DateTruncProcessor::new));
// math
entries.add(new Entry(Processor.class, BinaryMathProcessor.NAME, BinaryMathProcessor::new));
entries.add(new Entry(Processor.class, BinaryOptionalMathProcessor.NAME, BinaryOptionalMathProcessor::new));

70
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeParse.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Expressions;
import org.elasticsearch.xpack.ql.expression.function.scalar.BinaryScalarFunction;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.type.DataType;
import org.elasticsearch.xpack.ql.type.DataTypes;
import java.time.ZoneId;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString;
import static org.elasticsearch.xpack.ql.type.DateUtils.UTC;
public class DateTimeParse extends BinaryDateTimeFunction {
public DateTimeParse(Source source, Expression timestamp, Expression pattern) {
super(source, timestamp, pattern, UTC);
}
@Override
public DataType dataType() {
return DataTypes.DATETIME;
}
@Override
protected TypeResolution resolveType() {
TypeResolution resolution = isString(left(), sourceText(), Expressions.ParamOrdinal.FIRST);
if (resolution.unresolved()) {
return resolution;
}
resolution = isString(right(), sourceText(), Expressions.ParamOrdinal.SECOND);
if (resolution.unresolved()) {
return resolution;
}
return TypeResolution.TYPE_RESOLVED;
}
@Override
protected BinaryScalarFunction replaceChildren(Expression timestamp, Expression pattern) {
return new DateTimeParse(source(), timestamp, pattern);
}
@Override
protected NodeInfo<? extends Expression> info() {
return NodeInfo.create(this, DateTimeParse::new, left(), right());
}
@Override
protected String scriptMethodName() {
return "dateTimeParse";
}
@Override
public Object fold() {
return DateTimeParseProcessor.process(left().fold(), right().fold());
}
@Override
protected Pipe createPipe(Pipe timestamp, Pipe pattern, ZoneId zoneId) {
return new DateTimeParsePipe(source(), this, timestamp, pattern);
}
}

36
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeParsePipe.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import java.time.ZoneId;
public class DateTimeParsePipe extends BinaryDateTimePipe {
public DateTimeParsePipe(Source source, Expression expression, Pipe left, Pipe right) {
super(source, expression, left, right, null);
}
@Override
protected NodeInfo<DateTimeParsePipe> info() {
return NodeInfo.create(this, DateTimeParsePipe::new, expression(), left(), right());
}
@Override
protected DateTimeParsePipe replaceChildren(Pipe left, Pipe right) {
return new DateTimeParsePipe(source(), expression(), left, right);
}
@Override
protected Processor makeProcessor(Processor left, Processor right, ZoneId zoneId) {
return new DateTimeParseProcessor(left, right);
}
}

104
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeParseProcessor.java Normal file
View File

@ -0,0 +1,104 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
import java.io.IOException;
import java.time.DateTimeException;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.TemporalAccessor;
import java.util.Locale;
import java.util.Objects;
import static org.elasticsearch.xpack.ql.util.DateUtils.UTC;
public class DateTimeParseProcessor extends BinaryDateTimeProcessor {
public static final String NAME = "dtparse";
public DateTimeParseProcessor(Processor source1, Processor source2) {
super(source1, source2, null);
}
public DateTimeParseProcessor(StreamInput in) throws IOException {
super(in);
}
/**
* Used in Painless scripting
*/
public static Object process(Object timestampStr, Object pattern) {
if (timestampStr == null || pattern == null) {
return null;
}
if (timestampStr instanceof String == false) {
throw new SqlIllegalArgumentException("A string is required; received [{}]", timestampStr);
}
if (pattern instanceof String == false) {
throw new SqlIllegalArgumentException("A string is required; received [{}]", pattern);
}
if (((String) timestampStr).isEmpty() || ((String) pattern).isEmpty()) {
return null;
}
try {
TemporalAccessor ta = DateTimeFormatter.ofPattern((String) pattern, Locale.ROOT)
.parseBest((String) timestampStr, ZonedDateTime::from, LocalDateTime::from);
if (ta instanceof LocalDateTime) {
return ZonedDateTime.ofInstant((LocalDateTime) ta, ZoneOffset.UTC, UTC);
} else {
return ta;
}
} catch (IllegalArgumentException | DateTimeException e) {
String msg = e.getMessage();
if (msg.contains("Unable to convert parsed text using any of the specified queries")) {
msg = "Unable to convert parsed text into [datetime]";
}
throw new SqlIllegalArgumentException(
"Invalid date/time string [{}] or pattern [{}] is received; {}",
timestampStr,
pattern,
msg
);
}
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
protected Object doProcess(Object timestamp, Object pattern) {
return process(timestamp, pattern);
}
@Override
public int hashCode() {
return Objects.hash(left(), right());
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
DateTimeParseProcessor other = (DateTimeParseProcessor) obj;
return Objects.equals(left(), other.left()) && Objects.equals(right(), other.right());
}
}

5
x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
View File

@ -15,6 +15,7 @@ import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateDiffP
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DatePartProcessor;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFormatProcessor;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFunction;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeParseProcessor;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTruncProcessor;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.NamedDateTimeProcessor.NameExtractor;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.NonIsoDateTimeProcessor.NonIsoDateTimeExtractor;
@ -292,6 +293,10 @@ public class InternalSqlScriptUtils extends InternalQlScriptUtils {
return (String) DateTimeFormatProcessor.process(asDateTime(dateTime), pattern, ZoneId.of(tzId));
}
public static Object dateTimeParse(String dateField, String pattern, String tzId) {
return DateTimeParseProcessor.process(dateField, pattern);
}
public static ZonedDateTime asDateTime(Object dateTime) {
return (ZonedDateTime) asDateTime(dateTime, false);
}

1
x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
View File

@ -130,6 +130,7 @@ class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalS
def dateTrunc(String, Object, String)
Integer datePart(String, Object, String)
String dateTimeFormat(Object, String, String)
def dateTimeParse(String, String, String)
IntervalDayTime intervalDayTime(String, String)
IntervalYearMonth intervalYearMonth(String, String)
ZonedDateTime asDateTime(Object)

16
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
View File

@ -337,6 +337,22 @@ public class VerifierErrorMessagesTests extends ESTestCase {
);
}
public void testDateTimeParseValidArgs() {
accept("SELECT DATETIME_PARSE(keyword, 'MM/dd/uuuu HH:mm:ss') FROM test");
accept("SELECT DATETIME_PARSE('04/07/2020 10:20:30 Europe/Berlin', 'MM/dd/uuuu HH:mm:ss VV') FROM test");
}
public void testDateTimeParseInvalidArgs() {
assertEquals(
"1:8: first argument of [DATETIME_PARSE(int, keyword)] must be [string], found value [int] type [integer]",
error("SELECT DATETIME_PARSE(int, keyword) FROM test")
);
assertEquals(
"1:8: second argument of [DATETIME_PARSE(keyword, int)] must be [string], found value [int] type [integer]",
error("SELECT DATETIME_PARSE(keyword, int) FROM test")
);
}
public void testValidDateTimeFunctionsOnTime() {
accept("SELECT HOUR_OF_DAY(CAST(date AS TIME)) FROM test");
accept("SELECT MINUTE_OF_HOUR(CAST(date AS TIME)) FROM test");

118
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeParsePipeTests.java Normal file
View File

@ -0,0 +1,118 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.BinaryPipe;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.tree.AbstractNodeTestCase;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.tree.SourceTests;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.function.Function;
import static org.elasticsearch.xpack.ql.expression.Expressions.pipe;
import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.randomStringLiteral;
import static org.elasticsearch.xpack.ql.tree.SourceTests.randomSource;
public class DateTimeParsePipeTests extends AbstractNodeTestCase<DateTimeParsePipe, Pipe> {
public static DateTimeParsePipe randomDateTimeParsePipe() {
return (DateTimeParsePipe) new DateTimeParse(randomSource(), randomStringLiteral(), randomStringLiteral()).makePipe();
}
@Override
protected DateTimeParsePipe randomInstance() {
return randomDateTimeParsePipe();
}
private Expression randomDateTimeParsePipeExpression() {
return randomDateTimeParsePipe().expression();
}
@Override
public void testTransform() {
// test transforming only the properties (source, expression),
// skipping the children (the two parameters of the binary function) which are tested separately
DateTimeParsePipe b1 = randomInstance();
Expression newExpression = randomValueOtherThan(b1.expression(), this::randomDateTimeParsePipeExpression);
DateTimeParsePipe newB = new DateTimeParsePipe(b1.source(), newExpression, b1.left(), b1.right());
assertEquals(newB, b1.transformPropertiesOnly(v -> Objects.equals(v, b1.expression()) ? newExpression : v, Expression.class));
DateTimeParsePipe b2 = randomInstance();
Source newLoc = randomValueOtherThan(b2.source(), SourceTests::randomSource);
newB = new DateTimeParsePipe(newLoc, b2.expression(), b2.left(), b2.right());
assertEquals(newB, b2.transformPropertiesOnly(v -> Objects.equals(v, b2.source()) ? newLoc : v, Source.class));
}
@Override
public void testReplaceChildren() {
DateTimeParsePipe b = randomInstance();
Pipe newLeft = pipe(((Expression) randomValueOtherThan(b.left(), FunctionTestUtils::randomDatetimeLiteral)));
Pipe newRight = pipe(((Expression) randomValueOtherThan(b.right(), FunctionTestUtils::randomStringLiteral)));
DateTimeParsePipe newB = new DateTimeParsePipe(b.source(), b.expression(), b.left(), b.right());
BinaryPipe transformed = newB.replaceChildren(newLeft, b.right());
assertEquals(transformed.left(), newLeft);
assertEquals(transformed.source(), b.source());
assertEquals(transformed.expression(), b.expression());
assertEquals(transformed.right(), b.right());
transformed = newB.replaceChildren(b.left(), newRight);
assertEquals(transformed.left(), b.left());
assertEquals(transformed.source(), b.source());
assertEquals(transformed.expression(), b.expression());
assertEquals(transformed.right(), newRight);
transformed = newB.replaceChildren(newLeft, newRight);
assertEquals(transformed.left(), newLeft);
assertEquals(transformed.source(), b.source());
assertEquals(transformed.expression(), b.expression());
assertEquals(transformed.right(), newRight);
}
@Override
protected DateTimeParsePipe mutate(DateTimeParsePipe instance) {
List<Function<DateTimeParsePipe, DateTimeParsePipe>> randoms = new ArrayList<>();
randoms.add(
f -> new DateTimeParsePipe(
f.source(),
f.expression(),
pipe(((Expression) randomValueOtherThan(f.left(), FunctionTestUtils::randomDatetimeLiteral))),
f.right()
)
);
randoms.add(
f -> new DateTimeParsePipe(
f.source(),
f.expression(),
f.left(),
pipe(((Expression) randomValueOtherThan(f.right(), FunctionTestUtils::randomStringLiteral)))
)
);
randoms.add(
f -> new DateTimeParsePipe(
f.source(),
f.expression(),
pipe(((Expression) randomValueOtherThan(f.left(), FunctionTestUtils::randomDatetimeLiteral))),
pipe(((Expression) randomValueOtherThan(f.right(), FunctionTestUtils::randomStringLiteral)))
)
);
return randomFrom(randoms).apply(instance);
}
@Override
protected DateTimeParsePipe copy(DateTimeParsePipe instance) {
return new DateTimeParsePipe(instance.source(), instance.expression(), instance.left(), instance.right());
}
}

130
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeParseProcessorTests.java Normal file
View File

@ -0,0 +1,130 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.expression.function.scalar.datetime;
import org.elasticsearch.common.io.stream.Writeable.Reader;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ql.expression.gen.processor.ConstantProcessor;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.sql.AbstractSqlWireSerializingTestCase;
import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
import java.time.ZoneId;
import static org.elasticsearch.xpack.ql.expression.Literal.NULL;
import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l;
import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.randomStringLiteral;
import static org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeTestUtils.dateTime;
public class DateTimeParseProcessorTests extends AbstractSqlWireSerializingTestCase<DateTimeParseProcessor> {
public static DateTimeParseProcessor randomDateTimeParseProcessor() {
return new DateTimeParseProcessor(
new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 128)),
new ConstantProcessor(randomRealisticUnicodeOfLengthBetween(0, 128))
);
}
@Override
protected DateTimeParseProcessor createTestInstance() {
return randomDateTimeParseProcessor();
}
@Override
protected Reader<DateTimeParseProcessor> instanceReader() {
return DateTimeParseProcessor::new;
}
@Override
protected DateTimeParseProcessor mutateInstance(DateTimeParseProcessor instance) {
return new DateTimeParseProcessor(
new ConstantProcessor(ESTestCase.randomRealisticUnicodeOfLength(128)),
new ConstantProcessor(ESTestCase.randomRealisticUnicodeOfLength(128))
);
}
public void testInvalidInputs() {
SqlIllegalArgumentException siae = expectThrows(
SqlIllegalArgumentException.class,
() -> new DateTimeParse(Source.EMPTY, l(10), randomStringLiteral()).makePipe().asProcessor().process(null)
);
assertEquals("A string is required; received [10]", siae.getMessage());
siae = expectThrows(
SqlIllegalArgumentException.class,
() -> new DateTimeParse(Source.EMPTY, randomStringLiteral(), l(20)).makePipe().asProcessor().process(null)
);
assertEquals("A string is required; received [20]", siae.getMessage());
siae = expectThrows(
SqlIllegalArgumentException.class,
() -> new DateTimeParse(Source.EMPTY, l("2020-04-07"), l("invalid")).makePipe().asProcessor().process(null)
);
assertEquals(
"Invalid date/time string [2020-04-07] or pattern [invalid] is received; Unknown pattern letter: i",
siae.getMessage()
);
siae = expectThrows(
SqlIllegalArgumentException.class,
() -> new DateTimeParse(Source.EMPTY, l("2020-04-07"), l("MM/dd")).makePipe().asProcessor().process(null)
);
assertEquals(
"Invalid date/time string [2020-04-07] or pattern [MM/dd] is received; Text '2020-04-07' could not be parsed at index 2",
siae.getMessage()
);
siae = expectThrows(
SqlIllegalArgumentException.class,
() -> new DateTimeParse(Source.EMPTY, l("07/05/2020"), l("dd/MM/uuuu")).makePipe().asProcessor().process(null)
);
assertEquals(
"Invalid date/time string [07/05/2020] or pattern [dd/MM/uuuu] is received; Unable to convert parsed text into [datetime]",
siae.getMessage()
);
siae = expectThrows(
SqlIllegalArgumentException.class,
() -> new DateTimeParse(Source.EMPTY, l("10:20:30.123456789"), l("HH:mm:ss.SSSSSSSSS")).makePipe().asProcessor().process(null)
);
assertEquals(
"Invalid date/time string [10:20:30.123456789] or pattern [HH:mm:ss.SSSSSSSSS] is received; "
+ "Unable to convert parsed text into [datetime]",
siae.getMessage()
);
}
public void testWithNulls() {
assertNull(new DateTimeParse(Source.EMPTY, randomStringLiteral(), NULL).makePipe().asProcessor().process(null));
assertNull(new DateTimeParse(Source.EMPTY, randomStringLiteral(), l("")).makePipe().asProcessor().process(null));
assertNull(new DateTimeParse(Source.EMPTY, NULL, randomStringLiteral()).makePipe().asProcessor().process(null));
assertNull(new DateTimeParse(Source.EMPTY, l(""), randomStringLiteral()).makePipe().asProcessor().process(null));
}
public void testParsing() {
ZoneId zoneId = ZoneId.of("America/Sao_Paulo");
assertEquals(
dateTime(2020, 4, 7, 10, 20, 30, 123000000),
new DateTimeParse(Source.EMPTY, l("07/04/2020 10:20:30.123"), l("dd/MM/uuuu HH:mm:ss.SSS")).makePipe()
.asProcessor()
.process(null)
);
assertEquals(
dateTime(2020, 4, 7, 10, 20, 30, 123456789, zoneId),
new DateTimeParse(Source.EMPTY, l("07/04/2020 10:20:30.123456789 America/Sao_Paulo"), l("dd/MM/uuuu HH:mm:ss.SSSSSSSSS VV"))
.makePipe()
.asProcessor()
.process(null)
);
assertEquals(
dateTime(2020, 4, 7, 10, 20, 30, 123456789, ZoneId.of("+05:30")),
new DateTimeParse(Source.EMPTY, l("07/04/2020 10:20:30.123456789 +05:30"), l("dd/MM/uuuu HH:mm:ss.SSSSSSSSS zz")).makePipe()
.asProcessor()
.process(null)
);
}
}

17
x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
View File

@ -461,6 +461,23 @@ public class QueryTranslatorTests extends ESTestCase {
assertEquals("[{v=date}, {v=YYYY_MM_dd}, {v=Z}, {v=2018_09_04}]", sc.script().params().toString());
}
public void testTranslateDateTimeParse_WhereClause_Painless() {
LogicalPlan p = plan("SELECT int FROM test WHERE DATETIME_PARSE(keyword, 'uuuu_MM_dd') = '2018-09-04'::date");
assertTrue(p instanceof Project);
assertTrue(p.children().get(0) instanceof Filter);
Expression condition = ((Filter) p.children().get(0)).condition();
assertFalse(condition.foldable());
QueryTranslation translation = QueryTranslator.toQuery(condition, false);
assertNull(translation.aggFilter);
assertTrue(translation.query instanceof ScriptQuery);
ScriptQuery sc = (ScriptQuery) translation.query;
assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalSqlScriptUtils.dateTimeParse(" +
"InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),InternalSqlScriptUtils.asDateTime(params.v3)))",
sc.script().toString()
);
assertEquals("[{v=keyword}, {v=uuuu_MM_dd}, {v=Z}, {v=2018-09-04T00:00:00.000Z}]", sc.script().params().toString());
}
public void testLikeOnInexact() {
LogicalPlan p = plan("SELECT * FROM test WHERE some.string LIKE '%a%'");
assertTrue(p instanceof Project);