From b15f27f6a6bed3c349169a7c928e6c5460548bdb Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Mon, 17 Dec 2018 13:36:48 +0200 Subject: [PATCH] SQL: Scripting support for casting functions CAST and CONVERT (#36640) --- .../sql/qa/src/main/resources/agg.csv-spec | 66 +++++++++++++++++-- .../sql/expression/function/scalar/Cast.java | 16 +++++ .../whitelist/InternalSqlScriptUtils.java | 8 +++ .../xpack/sql/plugin/sql_whitelist.txt | 5 ++ .../xpack/sql/planner/QueryFolderTests.java | 8 +-- 5 files changed, 93 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec index 5d1e59ef7a2..d4837bfdafc 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec @@ -130,12 +130,66 @@ SELECT MAX(languages) max, MIN(languages) min, SUM(languages) sum, AVG(languages FROM test_emp GROUP BY languages ORDER BY languages ASC LIMIT 5; max:bt | min:bt | sum:bt | avg:d | percent:d | percent_rank:d| kurtosis:d | skewness:d ----------------+---------------+---------------+---------------+---------------+---------------+---------------+--------------- -null |null |null |null |null |null |null |null -1 |1 |15 |1 |1.0 |100.0 |NaN |NaN -2 |2 |38 |2 |2.0 |100.0 |NaN |NaN -3 |3 |51 |3 |3.0 |100.0 |NaN |NaN -4 |4 |72 |4 |4.0 |0.0 |NaN |NaN +---------------+---------------+---------------+--------------+---------------+---------------+---------------+--------------- +null |null |null |null |null |null |null |null +1 |1 |15 |1 |1.0 |100.0 |NaN |NaN +2 |2 |38 |2 |2.0 |100.0 |NaN |NaN +3 |3 |51 |3 |3.0 |100.0 |NaN |NaN +4 |4 |72 |4 |4.0 |0.0 |NaN |NaN +; + +aggByComplexCastedValue +SELECT CONVERT(CONCAT(LTRIM(CONVERT("emp_no", SQL_VARCHAR)), LTRIM(CONVERT("languages", SQL_VARCHAR))), SQL_BIGINT) AS "TEMP" +FROM "test_emp" GROUP BY "TEMP" ORDER BY "TEMP" LIMIT 20; + + TEMP:l +--------------- +10020 +10021 +10022 +10023 +10024 +10025 +10026 +10027 +10028 +10029 +100012 +100025 +100034 +100045 +100051 +100063 +100074 +100082 +100091 +100104 +; + +aggAndOrderByCastedValue +SELECT CHAR_LENGTH(SPACE(CAST(languages AS SMALLINT))), COUNT(*) FROM test_emp GROUP BY 1 ORDER BY 1 DESC; + +CHAR_LENGTH(SPACE(CAST(languages AS SMALLINT))):i| COUNT(1):l +-------------------------------------------------+--------------- +5 |21 +4 |18 +3 |17 +2 |19 +1 |15 +null |10 +; + +aggAndOrderByCastedFunctionValue +SELECT ROUND(SQRT(CAST(EXP(languages) AS SMALLINT)), 2), COUNT(*) FROM test_emp GROUP BY 1 ORDER BY 1 DESC; + +ROUND(SQRT(CAST(EXP(languages) AS SMALLINT)),2):d| COUNT(1):l +-------------------------------------------------+--------------- +12.17 |21 +7.42 |18 +4.47 |17 +2.65 |19 +1.73 |15 +null |10 ; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Cast.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Cast.java index 5c874cc7667..d4265d123e8 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Cast.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Cast.java @@ -7,14 +7,18 @@ package org.elasticsearch.xpack.sql.expression.function.scalar; import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.gen.processor.Processor; +import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.tree.NodeInfo; import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.type.DataTypeConversion; import org.elasticsearch.xpack.sql.type.DataTypes; +import java.util.Locale; import java.util.Objects; +import static org.elasticsearch.xpack.sql.expression.gen.script.ParamsBuilder.paramsBuilder; + public class Cast extends UnaryScalarFunction { private final DataType dataType; @@ -74,6 +78,18 @@ public class Cast extends UnaryScalarFunction { return new CastProcessor(DataTypeConversion.conversionFor(from(), to())); } + @Override + public ScriptTemplate asScript() { + ScriptTemplate fieldAsScript = asScript(field()); + return new ScriptTemplate( + formatTemplate(String.format(Locale.ROOT, "{sql}.cast(%s,{})", fieldAsScript.template())), + paramsBuilder() + .script(fieldAsScript.params()) + .variable(dataType.name()) + .build(), + dataType()); + } + @Override public int hashCode() { return Objects.hash(super.hashCode(), dataType); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java index b107598710c..cdc773a91af 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.Bina import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.InProcessor; import org.elasticsearch.xpack.sql.expression.predicate.regex.RegexProcessor.RegexOperation; import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DataTypeConversion; import org.elasticsearch.xpack.sql.util.DateUtils; import org.elasticsearch.xpack.sql.util.StringUtils; @@ -458,4 +459,11 @@ public final class InternalSqlScriptUtils { public static String ucase(String s) { return (String) StringOperation.UCASE.apply(s); } + + // + // Casting + // + public static Object cast(Object value, String typeName) { + return DataTypeConversion.convert(value, DataType.fromTypeName(typeName)); + } } diff --git a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt index b5b19004eee..4e9fc1475e3 100644 --- a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt +++ b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt @@ -130,4 +130,9 @@ class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalS String space(Number) String substring(String, Number, Number) String ucase(String) + +# +# Casting +# + def cast(Object, String) } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java index 6a6a1e2dd8e..617a4634826 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java @@ -240,10 +240,10 @@ public class QueryFolderTests extends ESTestCase { assertEquals(EsQueryExec.class, p.getClass()); EsQueryExec ee = (EsQueryExec) p; assertThat(ee.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""), - endsWith("{\"script\":{" + - "\"source\":\"InternalSqlScriptUtils.docValue(doc,params.v0)\",\"lang\":\"painless\"," + - "\"params\":{\"v0\":\"keyword\"}},\"missing_bucket\":true," + - "\"value_type\":\"ip\",\"order\":\"asc\"}}}]}}}")); + endsWith("{\"script\":{\"source\":\"InternalSqlScriptUtils.cast(" + + "InternalSqlScriptUtils.docValue(doc,params.v0),params.v1)\"," + + "\"lang\":\"painless\",\"params\":{\"v0\":\"keyword\",\"v1\":\"IP\"}}," + + "\"missing_bucket\":true,\"value_type\":\"ip\",\"order\":\"asc\"}}}]}}}")); assertEquals(2, ee.output().size()); assertThat(ee.output().get(0).toString(), startsWith("COUNT(1){a->")); assertThat(ee.output().get(1).toString(), startsWith("a{s->"));